pi-mega-compact 0.8.21 → 0.8.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +1 -1
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -14,6 +14,7 @@ import { decompressSmart } from "../src/store/compression.js";
|
|
|
14
14
|
import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
|
|
15
15
|
import { type MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
|
|
16
16
|
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
17
|
+
import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
|
|
17
18
|
import type { MegaConfig } from "./mega-config.js";
|
|
18
19
|
|
|
19
20
|
/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
|
|
@@ -95,9 +96,9 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
|
|
|
95
96
|
const pct = usage?.percent != null ? `${usage.percent}%` : "n/a";
|
|
96
97
|
const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
|
|
97
98
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
98
|
-
const st = runtime.store
|
|
99
|
-
const repo = runtime.store
|
|
100
|
-
const di = runtime.store
|
|
99
|
+
const st = vectorStats(runtime.store, sid);
|
|
100
|
+
const repo = vectorRepoStats(runtime.store);
|
|
101
|
+
const di = vectorDataInvariant(runtime.store);
|
|
101
102
|
const fmtB = (b: number) =>
|
|
102
103
|
b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
|
|
103
104
|
b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
|
|
@@ -20,6 +20,7 @@ import { isMegaCache } from "../../src/game/scoring.js";
|
|
|
20
20
|
import { resolveRepoRoot } from "../mega-config.js";
|
|
21
21
|
import { classifyError } from "./error-classifier.js";
|
|
22
22
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
23
|
+
import { vectorStats } from "../../src/vectorStore.js";
|
|
23
24
|
|
|
24
25
|
/** Register agent/turn tracking event handlers. */
|
|
25
26
|
export function registerAgentHandlers(
|
|
@@ -244,7 +245,7 @@ export function registerAgentHandlers(
|
|
|
244
245
|
try {
|
|
245
246
|
if (runtime.getCachedGameState().game_mode_on) {
|
|
246
247
|
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
247
|
-
const st = runtime.store
|
|
248
|
+
const st = vectorStats(runtime.store, runtime.rt.sessionId);
|
|
248
249
|
const cachePct = st.dedupHitRate * 100;
|
|
249
250
|
const modelId = runtime.currentModel?.modelId ?? "unknown";
|
|
250
251
|
recordScore(runtime.currentStateDir, {
|
|
@@ -110,6 +110,34 @@ export function registerCompactHandlers(
|
|
|
110
110
|
activeAgents: runtime.activeAgents,
|
|
111
111
|
});
|
|
112
112
|
if (!config.auto) return {}; // let pi run its own native compaction
|
|
113
|
+
|
|
114
|
+
// S38.5: COMPACT-DEDUP RACE GUARD — pi's _runAutoCompaction fires
|
|
115
|
+
// session_before_compact from within the agent loop (after agent_end,
|
|
116
|
+
// _handlePostAgentRun → _checkCompaction → _runAutoCompaction). If a
|
|
117
|
+
// compaction just completed (within the cooldown window), pi's
|
|
118
|
+
// prepareCompaction may still see the last entry as a compaction but
|
|
119
|
+
// the extension's session_compact handler hasn't been called yet, OR
|
|
120
|
+
// the setTimeout deferred ctx.compact() from the agent_end handler is
|
|
121
|
+
// about to race. Skip the durable trim and let pi run its own compact
|
|
122
|
+
// (or no-op if the branch is already compacted). Matches the cooldown
|
|
123
|
+
// in agent-handlers.ts so both call sites are consistent.
|
|
124
|
+
const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
|
|
125
|
+
const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
126
|
+
if (sinceCompact < cooldownMs) {
|
|
127
|
+
runtime.logger.info("before-compact-skip-recent", {
|
|
128
|
+
sessionId: runtime.rt.sessionId,
|
|
129
|
+
reason: event.reason,
|
|
130
|
+
sinceCompactMs: sinceCompact,
|
|
131
|
+
cooldownMs,
|
|
132
|
+
});
|
|
133
|
+
runtime.diagBeforeCompactSupplied++;
|
|
134
|
+
const fb = fallbackCompaction(event);
|
|
135
|
+
if (fb) {
|
|
136
|
+
return { compaction: fb.compaction };
|
|
137
|
+
}
|
|
138
|
+
return {};
|
|
139
|
+
}
|
|
140
|
+
|
|
113
141
|
try {
|
|
114
142
|
const result = driveNativeCompaction(event, runtime, config);
|
|
115
143
|
if (result && result.compaction.summary?.trim()) {
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
doRecallAsync,
|
|
18
18
|
} from "../mega-pipeline.js";
|
|
19
19
|
import { recallMemoriesAndInline } from "../../src/recall.js";
|
|
20
|
+
import { vectorStats } from "../../src/vectorStore.js";
|
|
20
21
|
import type { MegaConfig } from "../mega-config.js";
|
|
21
22
|
|
|
22
23
|
/** Register session lifecycle event handlers. */
|
|
@@ -50,7 +51,7 @@ export function registerSessionHandlers(
|
|
|
50
51
|
if (config.autoInline) {
|
|
51
52
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
52
53
|
const query = recentUserQuery(ctx);
|
|
53
|
-
if (query && runtime.store
|
|
54
|
+
if (query && vectorStats(runtime.store, sid).checkpointCount > 0) {
|
|
54
55
|
// S17: use the async variant on resume so cross-repo HNSW recall can
|
|
55
56
|
// augment when this repo's store is thin. session_start is an async-safe
|
|
56
57
|
// point (unlike the mid-turn context handler, which stays sync).
|
|
@@ -27,6 +27,7 @@ import { runRaptor } from "../../src/dedup/raptor/index.js";
|
|
|
27
27
|
import { loadDedupConfig } from "../../src/config/dedup.js";
|
|
28
28
|
import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
|
|
29
29
|
import { runMemoryReview } from "./memory-review.js";
|
|
30
|
+
import { vectorList } from "../../src/vectorStore.js";
|
|
30
31
|
|
|
31
32
|
export type RunCompactResult =
|
|
32
33
|
| { skipped: true }
|
|
@@ -212,7 +213,7 @@ function doCompact(
|
|
|
212
213
|
if (config.raptorEnabled && !result.deduped) {
|
|
213
214
|
try {
|
|
214
215
|
const dd = loadDedupConfig();
|
|
215
|
-
const all = runtime.store
|
|
216
|
+
const all = vectorList(runtime.store, sid);
|
|
216
217
|
const leaves = all.map((cp) => ({
|
|
217
218
|
id: cp.checkpointId,
|
|
218
219
|
messages: [],
|
|
@@ -248,7 +249,7 @@ function doCompact(
|
|
|
248
249
|
// Non-fatal: a WASM init failure degrades to the sync scan silently.
|
|
249
250
|
if (!result.deduped) {
|
|
250
251
|
try {
|
|
251
|
-
const all = runtime.store
|
|
252
|
+
const all = vectorList(runtime.store, sid);
|
|
252
253
|
const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
|
|
253
254
|
if (latest?.embedding) {
|
|
254
255
|
void indexUpsertEmbedding(
|
|
@@ -14,7 +14,7 @@ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
|
14
14
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { appendFileSync, mkdirSync } from "node:fs";
|
|
17
|
-
import { VectorStore } from "../../src/vectorStore.js";
|
|
17
|
+
import { VectorStore, vectorStats, vectorRepoStats, vectorDataInvariant } from "../../src/vectorStore.js";
|
|
18
18
|
import { toEngineMessages } from "../../src/adapt.js";
|
|
19
19
|
import { normalizeSessionId } from "../../src/store.js";
|
|
20
20
|
import { Logger } from "../../src/log.js";
|
|
@@ -365,8 +365,8 @@ export class MegaRuntime {
|
|
|
365
365
|
// never break the per-repo compaction path. Runs only on repo-switch
|
|
366
366
|
// (this branch), so it's infrequent — not per-context-event.
|
|
367
367
|
try {
|
|
368
|
-
const repo = this.store
|
|
369
|
-
const di = this.store
|
|
368
|
+
const repo = vectorRepoStats(this.store);
|
|
369
|
+
const di = vectorDataInvariant(this.store);
|
|
370
370
|
const root = key !== dir ? key : (resolveRepoRoot(cwd ?? dir) ?? dir);
|
|
371
371
|
upsertRepoRegistry({
|
|
372
372
|
repoRoot: root,
|
|
@@ -404,9 +404,9 @@ export class MegaRuntime {
|
|
|
404
404
|
return;
|
|
405
405
|
}
|
|
406
406
|
const perfT0 = performance.now();
|
|
407
|
-
const st = this.store
|
|
408
|
-
const repo = this.store
|
|
409
|
-
const di = this.store
|
|
407
|
+
const st = vectorStats(this.store, this.rt.sessionId);
|
|
408
|
+
const repo = vectorRepoStats(this.store);
|
|
409
|
+
const di = vectorDataInvariant(this.store);
|
|
410
410
|
// Live + store-wide cache-hit / compaction counters for the dashboard.
|
|
411
411
|
const ds = getDedupStats(this.currentStateDir);
|
|
412
412
|
const cacheHitsTotal = ds.deduped + getRecallInjected(this.currentStateDir);
|
|
@@ -555,7 +555,7 @@ export class MegaRuntime {
|
|
|
555
555
|
total: getCompactCount(this.currentStateDir),
|
|
556
556
|
},
|
|
557
557
|
timeSaved: {
|
|
558
|
-
compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(this.store
|
|
558
|
+
compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(vectorRepoStats(this.store).tokensSaved) },
|
|
559
559
|
cacheHit: { sessionSec: sec(this.rt.cacheHitTokens), totalSec: sec(cacheHitsTotalTokens) },
|
|
560
560
|
},
|
|
561
561
|
model,
|
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
type CompactResult,
|
|
23
23
|
} from "../src/engine.js";
|
|
24
24
|
import { recallAndInline, recallMemoriesAndInline, type RecallInjectResult } from "../src/recall.js";
|
|
25
|
-
import { VectorStore } from "../src/vectorStore.js";
|
|
25
|
+
import { VectorStore, vectorStats } from "../src/vectorStore.js";
|
|
26
26
|
import type { EngineMessage } from "../src/types.js";
|
|
27
27
|
|
|
28
28
|
// ---------------------------------------------------------------------------
|
|
@@ -260,7 +260,7 @@ export default definePluginEntry({
|
|
|
260
260
|
const sessionId = (args as Record<string, string>)?.sessionId ?? "global";
|
|
261
261
|
|
|
262
262
|
try {
|
|
263
|
-
const stats = store
|
|
263
|
+
const stats = vectorStats(store, sessionId);
|
|
264
264
|
const parts: string[] = [
|
|
265
265
|
`**Mega Compact Status**`,
|
|
266
266
|
`Session: ${sessionId}`,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.22",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-2-Clause",
|
|
@@ -12,7 +12,7 @@ import assert from "node:assert/strict";
|
|
|
12
12
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
13
13
|
import { tmpdir } from "node:os";
|
|
14
14
|
import { join } from "node:path";
|
|
15
|
-
import { VectorStore } from "../../vectorStore.js";
|
|
15
|
+
import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
|
|
16
16
|
import { runRaptor } from "./index.js";
|
|
17
17
|
import { compactSession } from "../../engine.js";
|
|
18
18
|
import { Logger } from "../../log.js";
|
|
@@ -44,11 +44,11 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
44
44
|
|
|
45
45
|
// No tree yet → flat search only, returns hits, no RAPTOR coverage.
|
|
46
46
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
|
|
47
|
-
const flat = s
|
|
47
|
+
const flat = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
48
48
|
assert.ok(flat.length > 0, "flat search returns hits");
|
|
49
49
|
|
|
50
50
|
// Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
|
|
51
|
-
const all = s
|
|
51
|
+
const all = vectorList(s, SESS);
|
|
52
52
|
const leaves = all.map((cp) => ({
|
|
53
53
|
id: cp.checkpointId,
|
|
54
54
|
messages: [],
|
|
@@ -60,7 +60,7 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
60
60
|
|
|
61
61
|
// With the tree live + RAPTOR_ENABLED, search still returns hits and now
|
|
62
62
|
// exercises the RAPTOR-served path without regression.
|
|
63
|
-
const withTree = s
|
|
63
|
+
const withTree = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
64
64
|
assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
|
|
65
65
|
// Every returned hit is a real checkpoint in the session.
|
|
66
66
|
for (const h of withTree) {
|
|
@@ -72,7 +72,7 @@ test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
|
|
|
72
72
|
const stateDir = join(baseTmp, `run-${counter++}`);
|
|
73
73
|
const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
|
|
74
74
|
compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
|
|
75
|
-
const r = s
|
|
75
|
+
const r = vectorSearch(s, SESS, "only one topic", 3);
|
|
76
76
|
assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
|
|
77
77
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
|
|
78
78
|
});
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore, L2_ENABLED } from "../vectorStore.js";
|
|
6
|
+
import { VectorStore, L2_ENABLED, vectorSemDedup, vectorList, vectorSearch } from "../vectorStore.js";
|
|
7
7
|
import { mmrRerank } from "./mmr.js";
|
|
8
8
|
import { topK } from "./topk.js";
|
|
9
9
|
import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
|
|
@@ -79,7 +79,7 @@ test("L2_ENABLED defaults true; search still returns hits", () => {
|
|
|
79
79
|
assert.equal(L2_ENABLED, true);
|
|
80
80
|
const s = store();
|
|
81
81
|
s.add({ sessionId: "sess_l2", summary: "investigated the parser", regionText: "investigated src/parser.ts and added a tokenizer", timestamp: 1 });
|
|
82
|
-
const hits = s
|
|
82
|
+
const hits = vectorSearch(s, "sess_l2", "src/parser.ts tokenizer", 3);
|
|
83
83
|
assert.ok(hits.length >= 1);
|
|
84
84
|
});
|
|
85
85
|
|
|
@@ -122,14 +122,14 @@ test("semDedup marks redundant near-identical rows 'removed' and search excludes
|
|
|
122
122
|
{ id: "chkpt_001", text: "the cache stores parsed ast nodes for fast lookup", tok: 100 },
|
|
123
123
|
{ id: "chkpt_002", text: "the cache stores parsed ast nodes for fast lookup and reuse", tok: 900 },
|
|
124
124
|
]);
|
|
125
|
-
const removed = s
|
|
125
|
+
const removed = vectorSemDedup(s,"sess_sd", 0.85);
|
|
126
126
|
assert.equal(removed, 1);
|
|
127
|
-
const st = s
|
|
127
|
+
const st = vectorList(s,"sess_sd");
|
|
128
128
|
const dropped = st.find((c) => c.dedupStatus === "removed");
|
|
129
129
|
assert.ok(dropped);
|
|
130
130
|
assert.equal(dropped.checkpointId, "chkpt_001"); // lower tokenEstimate removed
|
|
131
131
|
// Search excludes the removed row (only one active remains).
|
|
132
|
-
const hits = s
|
|
132
|
+
const hits = vectorSearch(s, "sess_sd", "cache parsed ast nodes", 5);
|
|
133
133
|
assert.equal(hits.length, 1);
|
|
134
134
|
});
|
|
135
135
|
|
|
@@ -140,8 +140,8 @@ test("semDedup is idempotent (re-run removes nothing new)", () => {
|
|
|
140
140
|
{ id: "chkpt_001", text: "identical region text for the dedup job now", tok: 100 },
|
|
141
141
|
{ id: "chkpt_002", text: "identical region text for the dedup job right now", tok: 200 },
|
|
142
142
|
]);
|
|
143
|
-
const first = s
|
|
144
|
-
const second = s
|
|
143
|
+
const first = vectorSemDedup(s,"sess_sd2", 0.85);
|
|
144
|
+
const second = vectorSemDedup(s,"sess_sd2", 0.85);
|
|
145
145
|
assert.equal(first, 1);
|
|
146
146
|
assert.equal(second, 0);
|
|
147
147
|
});
|
package/src/dedup-engine.test.ts
CHANGED
|
@@ -17,7 +17,7 @@ import fs from "fs";
|
|
|
17
17
|
import path from "path";
|
|
18
18
|
import os from "os";
|
|
19
19
|
import { compactSession } from "./engine.js";
|
|
20
|
-
import { VectorStore } from "./vectorStore.js";
|
|
20
|
+
import { VectorStore, vectorList, vectorStats, vectorWasInjected, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
|
|
21
21
|
import { extractiveSummarize } from "./extractive.js";
|
|
22
22
|
import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
|
|
23
23
|
import { autoCompactCheck } from "./compact.js";
|
|
@@ -181,7 +181,7 @@ describe("Dedupe Levels", () => {
|
|
|
181
181
|
const r2 = compactFull(s, SESS, [makeMsg("user", region)]);
|
|
182
182
|
assert.equal(r2.deduped, true);
|
|
183
183
|
assert.equal(r2.checkpointId, r1.checkpointId);
|
|
184
|
-
assert.equal(s
|
|
184
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
185
185
|
});
|
|
186
186
|
|
|
187
187
|
it("L0 only: distinct content stored twice creates two checkpoints", () => {
|
|
@@ -194,7 +194,7 @@ describe("Dedupe Levels", () => {
|
|
|
194
194
|
assert.equal(r1.deduped, false);
|
|
195
195
|
assert.equal(r2.deduped, false);
|
|
196
196
|
assert.notEqual(r1.checkpointId, r2.checkpointId);
|
|
197
|
-
assert.equal(s
|
|
197
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
198
198
|
});
|
|
199
199
|
|
|
200
200
|
it("L1 only: one-word variants collapse; major rewrites do not", () => {
|
|
@@ -209,11 +209,11 @@ describe("Dedupe Levels", () => {
|
|
|
209
209
|
|
|
210
210
|
const r2 = s.add({ sessionId: SESS, summary: "migration", regionText: variant, timestamp: 2 });
|
|
211
211
|
assert.equal(r2.deduped, true, "one-word variant should be collapsed by L1");
|
|
212
|
-
assert.equal(s
|
|
212
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
213
213
|
|
|
214
214
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: rewrite, timestamp: 3 });
|
|
215
215
|
assert.equal(r3.deduped, false, "major rewrite should not be collapsed by L1");
|
|
216
|
-
assert.equal(s
|
|
216
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
217
217
|
});
|
|
218
218
|
|
|
219
219
|
it("L2 only: semantic paraphrases collapse; unrelated topics do not", () => {
|
|
@@ -233,11 +233,11 @@ describe("Dedupe Levels", () => {
|
|
|
233
233
|
|
|
234
234
|
const r2 = s.add({ sessionId: SESS, summary: "auth paraphrase", regionText: paraphrase, timestamp: 2 });
|
|
235
235
|
assert.equal(r2.deduped, true, "semantic paraphrase should be collapsed by L2");
|
|
236
|
-
assert.equal(s
|
|
236
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
237
237
|
|
|
238
238
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: unrelated, timestamp: 3 });
|
|
239
239
|
assert.equal(r3.deduped, false, "unrelated topic should not be collapsed by L2");
|
|
240
|
-
assert.equal(s
|
|
240
|
+
assert.equal(vectorList(s,SESS).length, 2);
|
|
241
241
|
});
|
|
242
242
|
|
|
243
243
|
it("All tiers disabled: every store.add() with different region text creates a distinct checkpoint", () => {
|
|
@@ -254,7 +254,7 @@ describe("Dedupe Levels", () => {
|
|
|
254
254
|
assert.equal(r3.deduped, false);
|
|
255
255
|
assert.notEqual(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
256
256
|
assert.notEqual(r2.checkpoint.checkpointId, r3.checkpoint.checkpointId);
|
|
257
|
-
assert.equal(s
|
|
257
|
+
assert.equal(vectorList(s,SESS).length, 3);
|
|
258
258
|
});
|
|
259
259
|
|
|
260
260
|
it("Combined L0+L1+L2: layered behavior exact -> near -> semantic", () => {
|
|
@@ -284,8 +284,8 @@ describe("Dedupe Levels", () => {
|
|
|
284
284
|
okReason(r4.dedupReason, ["contentSimilarity", "l1MinHash"]);
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
-
assert.ok(s
|
|
288
|
-
assert.ok(s
|
|
287
|
+
assert.ok(vectorList(s,SESS).length >= 1, "layered dedup keeps at least one checkpoint");
|
|
288
|
+
assert.ok(vectorList(s,SESS).length <= 4, "layered dedup should not explode to many checkpoints");
|
|
289
289
|
});
|
|
290
290
|
});
|
|
291
291
|
|
|
@@ -354,18 +354,18 @@ describe("Compression / Store Stats", () => {
|
|
|
354
354
|
const dup = "duplicate topic about payment gateway integration";
|
|
355
355
|
compactFull(s, SESS, [makeMsg("user", dup)], 1);
|
|
356
356
|
|
|
357
|
-
const statsBefore = s
|
|
357
|
+
const statsBefore = vectorStats(s,SESS);
|
|
358
358
|
assert.ok(statsBefore.checkpointCount >= 1, "checkpointCount should be positive");
|
|
359
359
|
assert.ok(statsBefore.totalTokenEstimate >= 0, "totalTokenEstimate should be non-negative");
|
|
360
360
|
assert.equal(statsBefore.dedupHitRate, 0, "no injections yet => dedupHitRate 0");
|
|
361
361
|
assert.equal(statsBefore.injectedCount, 0, "no injections yet => injectedCount 0");
|
|
362
362
|
|
|
363
|
-
const hits = s
|
|
363
|
+
const hits = vectorSearch(s, SESS, "payment gateway", 5);
|
|
364
364
|
assert.ok(hits.length > 0, "should find the stored checkpoint");
|
|
365
365
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
366
366
|
|
|
367
|
-
s
|
|
368
|
-
const statsAfter = s
|
|
367
|
+
vectorMarkInjected(s,SESS, cpId);
|
|
368
|
+
const statsAfter = vectorStats(s,SESS);
|
|
369
369
|
assert.equal(statsAfter.injectedCount, 1, "injectedCount tracks markInjected");
|
|
370
370
|
if (statsAfter.checkpointCount > 0) {
|
|
371
371
|
assert.ok(
|
|
@@ -387,10 +387,10 @@ describe("Compression / Store Stats", () => {
|
|
|
387
387
|
compactFull(s, SESS, [makeMsg("user", "unique region for hit-rate measurement variant")], 1);
|
|
388
388
|
|
|
389
389
|
// Mark the first as injected.
|
|
390
|
-
const first = s
|
|
391
|
-
if (first) s
|
|
390
|
+
const first = vectorSearch(s, SESS, base, 1)[0]?.checkpoint.checkpointId;
|
|
391
|
+
if (first) vectorMarkInjected(s,SESS, first);
|
|
392
392
|
|
|
393
|
-
const stats = s
|
|
393
|
+
const stats = vectorStats(s,SESS);
|
|
394
394
|
assert.ok(stats.checkpointCount >= 1);
|
|
395
395
|
assert.ok(
|
|
396
396
|
stats.dedupHitRate > 0 || stats.checkpointCount === 1,
|
|
@@ -444,16 +444,16 @@ describe("Recall & Dedup Sentinel", () => {
|
|
|
444
444
|
const region = "manual sentinel tracking without recallAndInline";
|
|
445
445
|
compactFull(s, SESS, [makeMsg("user", region)]);
|
|
446
446
|
|
|
447
|
-
const hits = s
|
|
447
|
+
const hits = vectorSearch(s, SESS, "manual sentinel", 3);
|
|
448
448
|
assert.ok(hits.length > 0, "search should return checkpoint");
|
|
449
449
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
450
|
-
assert.equal(s
|
|
450
|
+
assert.equal(vectorWasInjected(s,SESS, cpId), false, "not yet injected");
|
|
451
451
|
|
|
452
|
-
s
|
|
453
|
-
assert.equal(s
|
|
452
|
+
vectorMarkInjected(s,SESS, cpId);
|
|
453
|
+
assert.equal(vectorWasInjected(s,SESS, cpId), true, "markInjected recorded");
|
|
454
454
|
|
|
455
|
-
const hits2 = s
|
|
456
|
-
(h) => !s
|
|
455
|
+
const hits2 = vectorSearch(s, SESS, "manual sentinel", 3).filter(
|
|
456
|
+
(h) => !vectorWasInjected(s,SESS, h.checkpoint.checkpointId),
|
|
457
457
|
);
|
|
458
458
|
assert.equal(hits2.length, 0, "filtered search excludes injected checkpoint");
|
|
459
459
|
});
|
|
@@ -469,7 +469,7 @@ describe("Edge Cases", () => {
|
|
|
469
469
|
const r = compactSession({ sessionId: SESS, messages: [], keepFrom: 0 }, s);
|
|
470
470
|
assert.equal(r.skipped, true);
|
|
471
471
|
assert.equal(r.summary, "");
|
|
472
|
-
assert.equal(s
|
|
472
|
+
assert.equal(vectorList(s,SESS).length, 0);
|
|
473
473
|
});
|
|
474
474
|
|
|
475
475
|
it("single message with keepFrom=0 returns skipped", () => {
|
|
@@ -479,7 +479,7 @@ describe("Edge Cases", () => {
|
|
|
479
479
|
s,
|
|
480
480
|
);
|
|
481
481
|
assert.equal(r.skipped, true);
|
|
482
|
-
assert.equal(s
|
|
482
|
+
assert.equal(vectorList(s,SESS).length, 0);
|
|
483
483
|
});
|
|
484
484
|
|
|
485
485
|
it("keepFrom at messages.length compacts all prior messages (verified behavior)", () => {
|
|
@@ -491,7 +491,7 @@ describe("Edge Cases", () => {
|
|
|
491
491
|
const r = compactSession({ sessionId: SESS, messages, keepFrom: messages.length }, s);
|
|
492
492
|
assert.equal(r.skipped, false);
|
|
493
493
|
assert.ok(r.checkpointId);
|
|
494
|
-
assert.equal(s
|
|
494
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
495
495
|
});
|
|
496
496
|
|
|
497
497
|
it("unicode and emoji messages store and retrieve intact", () => {
|
|
@@ -501,7 +501,7 @@ describe("Edge Cases", () => {
|
|
|
501
501
|
"日本語テキスト 日本語テキスト 👍🔥";
|
|
502
502
|
const r = compactFull(s, SESS, [makeMsg("user", text)], 1);
|
|
503
503
|
assert.equal(r.skipped, false);
|
|
504
|
-
const stored = s
|
|
504
|
+
const stored = vectorList(s,SESS)[0];
|
|
505
505
|
assert.ok(stored);
|
|
506
506
|
const recovered = Buffer.from(stored.compressedOriginal ?? Buffer.alloc(0));
|
|
507
507
|
assert.ok(
|
|
@@ -518,8 +518,8 @@ describe("Edge Cases", () => {
|
|
|
518
518
|
const r = compactFull(s, SESS, [makeMsg("user", big)], 1);
|
|
519
519
|
assert.equal(r.skipped, false);
|
|
520
520
|
assert.ok(r.checkpointId);
|
|
521
|
-
assert.equal(s
|
|
522
|
-
const stats = s
|
|
521
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
522
|
+
const stats = vectorStats(s,SESS);
|
|
523
523
|
assert.ok(stats.totalTokenEstimate > 0);
|
|
524
524
|
});
|
|
525
525
|
|
|
@@ -542,7 +542,7 @@ describe("Edge Cases", () => {
|
|
|
542
542
|
r.summary.includes("assistant"),
|
|
543
543
|
"summary should reference roles or tools",
|
|
544
544
|
);
|
|
545
|
-
assert.equal(s
|
|
545
|
+
assert.equal(vectorList(s,SESS).length, 1);
|
|
546
546
|
});
|
|
547
547
|
});
|
|
548
548
|
|