pi-mega-compact 0.8.24 → 0.8.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/extensions/mega-compact-s38.test.js +263 -14
- package/dist/extensions/mega-compact.js +15 -0
- package/dist/extensions/mega-config.js +3 -0
- package/dist/extensions/mega-events/agent-handlers.js +211 -26
- package/dist/extensions/mega-events/context-handler.js +45 -7
- package/dist/extensions/mega-events/error-classifier.js +125 -18
- package/dist/extensions/mega-pipeline/compact.js +24 -13
- package/dist/extensions/mega-pipeline/recall.js +31 -2
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime.js +58 -5
- package/dist/src/boundary.js +79 -43
- package/dist/src/boundary.test.js +119 -2
- package/dist/src/canary.js +10 -0
- package/dist/src/config/dedup.js +14 -0
- package/dist/src/config.js +3 -1
- package/dist/src/dedup/raptor/buildHistory.js +164 -0
- package/dist/src/dedup/raptor/buildHistory.test.js +292 -0
- package/dist/src/dedup/raptor/index.js +38 -0
- package/dist/src/dedup/raptor/multilevel-serve.test.js +229 -0
- package/dist/src/dedup/raptor/multilevel.js +17 -5
- package/dist/src/dedup/raptor/multilevel.test.js +36 -1
- package/dist/src/dedup/raptor/raptor.test.js +43 -0
- package/dist/src/dedup/raptor/retrieval.js +14 -2
- package/dist/src/dedup/raptor/retrieval.test.js +95 -0
- package/dist/src/dedup/raptor/serve-gate.test.js +298 -0
- package/dist/src/dedup/raptor/summarizer.js +1 -0
- package/dist/src/dedup/raptor/tree.js +16 -2
- package/dist/src/engine.js +18 -2
- package/dist/src/httpEmbedder.js +96 -6
- package/dist/src/httpEmbedder.test.js +277 -0
- package/dist/src/mechanical-fix.test.js +65 -0
- package/dist/src/raptor-inject-summaries.test.js +162 -0
- package/dist/src/recall.js +153 -24
- package/dist/src/recall.test.js +179 -4
- package/dist/src/store/sqlite/dedup-mirror.js +32 -15
- package/dist/src/store/sqlite/maintenance.js +2 -2
- package/dist/src/store/sqlite/mechanical-fix.test.js +146 -0
- package/dist/src/store/sqlite/memories.js +5 -5
- package/dist/src/store/sqlite/meta.js +1 -1
- package/dist/src/store/sqlite/raptor.js +56 -17
- package/dist/src/store/sqlite/raptor.test.js +106 -0
- package/dist/src/store/sqlite/schema.js +90 -1
- package/dist/src/store/sqlite/session-state.js +9 -3
- package/dist/src/store/sqlite/stats.js +9 -5
- package/dist/src/store/sqlite/turns.js +179 -0
- package/dist/src/store/sqlite/turns.test.js +183 -0
- package/dist/src/store/sqlite/utils.js +15 -4
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/store.js +2 -2
- package/dist/src/vector-search-cache.test.js +157 -0
- package/dist/src/vector-search.js +107 -15
- package/dist/src/vectorStore.js +36 -8
- package/extensions/mega-compact-s38.test.ts +259 -14
- package/extensions/mega-compact.ts +15 -0
- package/extensions/mega-config.ts +18 -0
- package/extensions/mega-dashboard.ts +10 -1
- package/extensions/mega-events/agent-handlers.ts +211 -26
- package/extensions/mega-events/context-handler.ts +43 -7
- package/extensions/mega-events/error-classifier.ts +125 -17
- package/extensions/mega-pipeline/compact.ts +28 -16
- package/extensions/mega-pipeline/recall.ts +34 -2
- package/extensions/mega-runtime/dashboard-snapshot.ts +8 -0
- package/extensions/mega-runtime/helpers.ts +25 -1
- package/extensions/mega-runtime/runtime-snapshot.ts +4 -0
- package/extensions/mega-runtime/runtime.ts +69 -23
- package/package.json +1 -1
- package/src/boundary.test.ts +128 -2
- package/src/boundary.ts +75 -39
- package/src/canary.ts +10 -0
- package/src/config/dedup.ts +25 -0
- package/src/config.ts +3 -1
- package/src/dedup/raptor/buildHistory.test.ts +353 -0
- package/src/dedup/raptor/buildHistory.ts +259 -0
- package/src/dedup/raptor/index.ts +38 -0
- package/src/dedup/raptor/multilevel-serve.test.ts +273 -0
- package/src/dedup/raptor/multilevel.test.ts +47 -0
- package/src/dedup/raptor/multilevel.ts +18 -8
- package/src/dedup/raptor/raptor.test.ts +59 -0
- package/src/dedup/raptor/retrieval.test.ts +118 -0
- package/src/dedup/raptor/retrieval.ts +14 -2
- package/src/dedup/raptor/serve-gate.test.ts +348 -0
- package/src/dedup/raptor/summarizer.ts +1 -0
- package/src/dedup/raptor/tree.ts +17 -2
- package/src/engine.ts +32 -3
- package/src/httpEmbedder.test.ts +286 -0
- package/src/httpEmbedder.ts +98 -8
- package/src/mechanical-fix.test.ts +70 -0
- package/src/raptor-inject-summaries.test.ts +228 -0
- package/src/recall.test.ts +220 -4
- package/src/recall.ts +462 -265
- package/src/store/sqlite/dedup-mirror.ts +35 -18
- package/src/store/sqlite/maintenance.ts +2 -2
- package/src/store/sqlite/mechanical-fix.test.ts +162 -0
- package/src/store/sqlite/memories.ts +5 -5
- package/src/store/sqlite/meta.ts +1 -1
- package/src/store/sqlite/raptor.test.ts +139 -0
- package/src/store/sqlite/raptor.ts +135 -81
- package/src/store/sqlite/schema.ts +90 -1
- package/src/store/sqlite/session-state.ts +9 -3
- package/src/store/sqlite/stats.ts +10 -8
- package/src/store/sqlite/turns.test.ts +218 -0
- package/src/store/sqlite/turns.ts +302 -0
- package/src/store/sqlite/utils.ts +14 -4
- package/src/store/sqlite.ts +1 -0
- package/src/store.ts +9 -2
- package/src/vector-search-cache.test.ts +190 -0
- package/src/vector-search.ts +273 -156
- package/src/vectorStore.ts +443 -382
- package/extensions/mega-runtime/reset-runtime.ts +0 -80
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turns.ts — S43 per-turn vector + conversation tracking.
|
|
3
|
+
*
|
|
4
|
+
* The relational spine for per-turn + per-conversation memories:
|
|
5
|
+
* - `turns`: one row per turn_end (metrics + epoch link + conversation id)
|
|
6
|
+
* - `turn_recall`: which checkpoints/cluster summaries were injected at a turn
|
|
7
|
+
* - `conversation_branches`: fork registry (parent → child at a turn point)
|
|
8
|
+
*
|
|
9
|
+
* `conversationId` (on SessionState) groups turns across pi session resumes;
|
|
10
|
+
* `/clear` or any fresh root generates a new one via `newConversationId()`.
|
|
11
|
+
*
|
|
12
|
+
* `forkConversation(parent, forkTurnId)` copies a parent conversation's
|
|
13
|
+
* recall state at `forkTurnId` into a fresh conversation's injected-set, so a
|
|
14
|
+
* forked session starts with exactly the context conversation X had at turn N
|
|
15
|
+
* (a recall-fork, not a live-window replay — see docs/specs/s48-per-turn-vector-tracking.md).
|
|
16
|
+
*
|
|
17
|
+
* No network. Pure SQLite (PREVENT-PI-004). All queries parameterized (PREVENT-002).
|
|
18
|
+
*/
|
|
19
|
+
import { randomBytes } from "node:crypto";
|
|
20
|
+
import { openStore, withTx } from "./utils.js";
|
|
21
|
+
import { getStateDir, normalizeSessionId } from "../../store.js";
|
|
22
|
+
import { loadSessionState, saveSessionState } from "./session-state.js";
|
|
23
|
+
/** Generate a new conversation id (`conv_` + 16 hex). */
|
|
24
|
+
export function newConversationId() {
|
|
25
|
+
return `conv_${randomBytes(8).toString("hex")}`;
|
|
26
|
+
}
|
|
27
|
+
function rowToTurn(r) {
|
|
28
|
+
return {
|
|
29
|
+
id: r.id,
|
|
30
|
+
conversationId: r.conversation_id,
|
|
31
|
+
sessionId: r.session_id,
|
|
32
|
+
turnIndex: r.turn_index,
|
|
33
|
+
role: r.role ?? null,
|
|
34
|
+
startedAt: r.started_at,
|
|
35
|
+
endedAt: r.ended_at ?? null,
|
|
36
|
+
ctxTokens: r.ctx_tokens ?? null,
|
|
37
|
+
ctxPercent: r.ctx_percent ?? null,
|
|
38
|
+
pressureBand: r.pressure_band ?? null,
|
|
39
|
+
modelId: r.model_id ?? null,
|
|
40
|
+
epochId: r.epoch_id ?? null,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Upsert a turn row at turn_end. Returns the turn id. `startedAt` defaults to
|
|
45
|
+
* now (the turn_start write can update it; turn_end sets ended_at + metrics).
|
|
46
|
+
* Idempotent on (session_id, turn_index) — re-upserting overwrites metrics.
|
|
47
|
+
*/
|
|
48
|
+
export function recordTurn(input, stateDir = getStateDir()) {
|
|
49
|
+
const db = openStore(stateDir);
|
|
50
|
+
const sid = normalizeSessionId(input.sessionId);
|
|
51
|
+
const startedAt = input.startedAt ?? Date.now();
|
|
52
|
+
db.prepare(`INSERT INTO turns (conversation_id, session_id, turn_index, role, started_at,
|
|
53
|
+
ended_at, ctx_tokens, ctx_percent, pressure_band, model_id, epoch_id)
|
|
54
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
55
|
+
ON CONFLICT(session_id, turn_index) DO UPDATE SET
|
|
56
|
+
conversation_id = excluded.conversation_id,
|
|
57
|
+
role = COALESCE(excluded.role, role),
|
|
58
|
+
started_at = COALESCE(excluded.started_at, started_at),
|
|
59
|
+
ended_at = COALESCE(excluded.ended_at, ended_at),
|
|
60
|
+
ctx_tokens = COALESCE(excluded.ctx_tokens, ctx_tokens),
|
|
61
|
+
ctx_percent = COALESCE(excluded.ctx_percent, ctx_percent),
|
|
62
|
+
pressure_band = COALESCE(excluded.pressure_band, pressure_band),
|
|
63
|
+
model_id = COALESCE(excluded.model_id, model_id),
|
|
64
|
+
epoch_id = COALESCE(excluded.epoch_id, epoch_id)`).run(input.conversationId, sid, input.turnIndex, input.role ?? null, startedAt, input.endedAt ?? null, input.ctxTokens ?? null, input.ctxPercent ?? null, input.pressureBand ?? null, input.modelId ?? null, input.epochId ?? null);
|
|
65
|
+
const row = db
|
|
66
|
+
.prepare("SELECT id FROM turns WHERE session_id = ? AND turn_index = ?")
|
|
67
|
+
.get(sid, input.turnIndex);
|
|
68
|
+
return row.id;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Record what was recalled at a turn. Called from recallAndInline with the
|
|
72
|
+
* resolved toInject list — each hit becomes a turn_recall row with its source
|
|
73
|
+
* path + score. RAPTOR cluster hits carry raptorLevel. Best-effort + non-fatal.
|
|
74
|
+
*/
|
|
75
|
+
export function recordTurnRecall(turnId, hits, stateDir = getStateDir()) {
|
|
76
|
+
if (hits.length === 0)
|
|
77
|
+
return;
|
|
78
|
+
const db = openStore(stateDir);
|
|
79
|
+
const stmt = db.prepare(`INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
|
|
80
|
+
VALUES (?, ?, ?, ?, ?)
|
|
81
|
+
ON CONFLICT(turn_id, checkpoint_id) DO UPDATE SET
|
|
82
|
+
score = excluded.score, source = excluded.source,
|
|
83
|
+
raptor_level = excluded.raptor_level`);
|
|
84
|
+
withTx(db, () => {
|
|
85
|
+
for (const h of hits) {
|
|
86
|
+
stmt.run(turnId, h.checkpointId, h.score, h.source, h.raptorLevel ?? null);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
/** Get a turn by conversation id + turn index (the lookup a fork uses). */
|
|
91
|
+
export function getTurn(conversationId, turnIndex, stateDir = getStateDir()) {
|
|
92
|
+
const db = openStore(stateDir);
|
|
93
|
+
const row = db
|
|
94
|
+
.prepare(`SELECT * FROM turns WHERE conversation_id = ? AND turn_index = ?`)
|
|
95
|
+
.get(conversationId, turnIndex);
|
|
96
|
+
return row ? rowToTurn(row) : null;
|
|
97
|
+
}
|
|
98
|
+
/** Get a turn by its global id. */
|
|
99
|
+
export function getTurnById(turnId, stateDir = getStateDir()) {
|
|
100
|
+
const db = openStore(stateDir);
|
|
101
|
+
const row = db.prepare("SELECT * FROM turns WHERE id = ?").get(turnId);
|
|
102
|
+
return row ? rowToTurn(row) : null;
|
|
103
|
+
}
|
|
104
|
+
/** All turn_recall rows for a turn (what was injected at that turn). */
|
|
105
|
+
export function listTurnRecall(turnId, stateDir = getStateDir()) {
|
|
106
|
+
const db = openStore(stateDir);
|
|
107
|
+
const rows = db
|
|
108
|
+
.prepare(`SELECT id, turn_id, checkpoint_id, score, source, raptor_level
|
|
109
|
+
FROM turn_recall WHERE turn_id = ? ORDER BY score DESC`)
|
|
110
|
+
.all(turnId);
|
|
111
|
+
return rows.map((r) => ({
|
|
112
|
+
id: r.id,
|
|
113
|
+
turnId: r.turn_id,
|
|
114
|
+
checkpointId: r.checkpoint_id,
|
|
115
|
+
score: r.score,
|
|
116
|
+
source: r.source,
|
|
117
|
+
raptorLevel: r.raptor_level ?? null,
|
|
118
|
+
}));
|
|
119
|
+
}
|
|
120
|
+
/** All turns in a conversation, ascending by turn_index. */
|
|
121
|
+
export function listConversationTurns(conversationId, stateDir = getStateDir()) {
|
|
122
|
+
const db = openStore(stateDir);
|
|
123
|
+
const rows = db
|
|
124
|
+
.prepare(`SELECT * FROM turns WHERE conversation_id = ? ORDER BY turn_index ASC`)
|
|
125
|
+
.all(conversationId);
|
|
126
|
+
return rows.map(rowToTurn);
|
|
127
|
+
}
|
|
128
|
+
/** Resolve a session's conversation id, generating + persisting one if none.
|
|
129
|
+
* A resumed session inherits its existing conversationId from session_state. */
|
|
130
|
+
export function ensureConversationId(sessionId, stateDir = getStateDir()) {
|
|
131
|
+
const st = loadSessionState(sessionId, stateDir);
|
|
132
|
+
if (st.conversationId)
|
|
133
|
+
return st.conversationId;
|
|
134
|
+
const conv = newConversationId();
|
|
135
|
+
saveSessionState(sessionId, {
|
|
136
|
+
...st,
|
|
137
|
+
conversationId: conv,
|
|
138
|
+
}, stateDir);
|
|
139
|
+
return conv;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Fork a conversation at `forkTurnId`: create a new conversation id, record the
|
|
143
|
+
* branch lineage, and return the parent's recall set at that turn (the
|
|
144
|
+
* checkpoint_ids + scores that were injected) so the caller can seed the forked
|
|
145
|
+
* session's injected-set with exactly that context.
|
|
146
|
+
*
|
|
147
|
+
* Returns the new conversation id + the recall-set rows to replay. The caller
|
|
148
|
+
* (the engine/extension) is responsible for marking those checkpoint_ids as
|
|
149
|
+
* injected in the new session's session_state so they're not re-recalled.
|
|
150
|
+
*/
|
|
151
|
+
export function forkConversation(parentConversationId, forkTurnId, stateDir = getStateDir()) {
|
|
152
|
+
const childId = newConversationId();
|
|
153
|
+
const db = openStore(stateDir);
|
|
154
|
+
withTx(db, () => {
|
|
155
|
+
db.prepare(`INSERT INTO conversation_branches
|
|
156
|
+
(conversation_id, parent_conversation_id, fork_turn_id, created_at)
|
|
157
|
+
VALUES (?, ?, ?, ?)
|
|
158
|
+
ON CONFLICT(conversation_id) DO NOTHING`).run(childId, parentConversationId, forkTurnId, Date.now());
|
|
159
|
+
});
|
|
160
|
+
// Replay set: the parent's injected checkpoints at the fork turn.
|
|
161
|
+
const recalled = listTurnRecall(forkTurnId, stateDir);
|
|
162
|
+
return { conversationId: childId, recalled };
|
|
163
|
+
}
|
|
164
|
+
/** Clear turn tracking rows for a session (tests / DR). */
|
|
165
|
+
export function clearTurns(sessionId, stateDir = getStateDir()) {
|
|
166
|
+
const db = openStore(stateDir);
|
|
167
|
+
const sid = normalizeSessionId(sessionId);
|
|
168
|
+
withTx(db, () => {
|
|
169
|
+
const turnIds = db
|
|
170
|
+
.prepare("SELECT id FROM turns WHERE session_id = ?")
|
|
171
|
+
.all(sid);
|
|
172
|
+
const ids = turnIds.map((t) => t.id);
|
|
173
|
+
if (ids.length > 0) {
|
|
174
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
175
|
+
db.prepare(`DELETE FROM turn_recall WHERE turn_id IN (${placeholders})`).run(...ids);
|
|
176
|
+
}
|
|
177
|
+
db.prepare("DELETE FROM turns WHERE session_id = ?").run(sid);
|
|
178
|
+
});
|
|
179
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turns.test.ts — S43 per-turn + conversation tracking tests.
|
|
3
|
+
*
|
|
4
|
+
* No network. Real stores with temp state dirs.
|
|
5
|
+
*/
|
|
6
|
+
import { test, beforeEach, afterEach } from "node:test";
|
|
7
|
+
import assert from "node:assert/strict";
|
|
8
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { openStore } from "./utils.js";
|
|
12
|
+
import { recordTurn, recordTurnRecall, getTurn, getTurnById, listTurnRecall, listConversationTurns, ensureConversationId, forkConversation, newConversationId, clearTurns, } from "./turns.js";
|
|
13
|
+
import { loadSessionState, saveSessionState } from "./session-state.js";
|
|
14
|
+
let tmpDir;
|
|
15
|
+
let counter = 0;
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
tmpDir = mkdtempSync(join(tmpdir(), "mc-turns-"));
|
|
18
|
+
});
|
|
19
|
+
afterEach(() => {
|
|
20
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
21
|
+
});
|
|
22
|
+
function stateDir() {
|
|
23
|
+
return join(tmpDir, `run-${counter++}`);
|
|
24
|
+
}
|
|
25
|
+
// ─── 1. recordTurn upserts + getTurn round-trip ────────────────────────────
|
|
26
|
+
test("S43-1: recordTurn inserts/updates + getTurn finds by conv+index", () => {
|
|
27
|
+
const sd = stateDir();
|
|
28
|
+
openStore(sd);
|
|
29
|
+
const conv = newConversationId();
|
|
30
|
+
const id = recordTurn({
|
|
31
|
+
conversationId: conv,
|
|
32
|
+
sessionId: "sess-1",
|
|
33
|
+
turnIndex: 3,
|
|
34
|
+
endedAt: 1000,
|
|
35
|
+
ctxTokens: 12000,
|
|
36
|
+
ctxPercent: 42.5,
|
|
37
|
+
pressureBand: "mid",
|
|
38
|
+
modelId: "claude-fable-5",
|
|
39
|
+
}, sd);
|
|
40
|
+
assert.ok(id > 0, "turn id returned");
|
|
41
|
+
const t = getTurn(conv, 3, sd);
|
|
42
|
+
assert.ok(t, "turn found");
|
|
43
|
+
assert.equal(t.conversationId, conv);
|
|
44
|
+
assert.equal(t.turnIndex, 3);
|
|
45
|
+
assert.equal(t.ctxTokens, 12000);
|
|
46
|
+
assert.equal(t.ctxPercent, 42.5);
|
|
47
|
+
assert.equal(t.pressureBand, "mid");
|
|
48
|
+
assert.equal(t.modelId, "claude-fable-5");
|
|
49
|
+
assert.equal(t.endedAt, 1000);
|
|
50
|
+
// Re-record (upsert) with additional metrics — COALESCE preserves started_at.
|
|
51
|
+
const id2 = recordTurn({
|
|
52
|
+
conversationId: conv,
|
|
53
|
+
sessionId: "sess-1",
|
|
54
|
+
turnIndex: 3,
|
|
55
|
+
startedAt: 900,
|
|
56
|
+
ctxTokens: 12500,
|
|
57
|
+
}, sd);
|
|
58
|
+
assert.equal(id2, id, "same turn id on upsert");
|
|
59
|
+
const t2 = getTurn(conv, 3, sd);
|
|
60
|
+
assert.equal(t2.ctxTokens, 12500, "metrics updated");
|
|
61
|
+
assert.equal(t2.startedAt, 900, "started_at set on second write");
|
|
62
|
+
});
|
|
63
|
+
// ─── 2. recordTurnRecall + listTurnRecall ───────────────────────────────────
|
|
64
|
+
test("S43-2: recordTurnRecall stores provenance + listTurnRecall returns it", () => {
|
|
65
|
+
const sd = stateDir();
|
|
66
|
+
openStore(sd);
|
|
67
|
+
const conv = newConversationId();
|
|
68
|
+
const turnId = recordTurn({
|
|
69
|
+
conversationId: conv,
|
|
70
|
+
sessionId: "sess-1",
|
|
71
|
+
turnIndex: 1,
|
|
72
|
+
}, sd);
|
|
73
|
+
recordTurnRecall(turnId, [
|
|
74
|
+
{ checkpointId: "chkpt_003", score: 0.91, source: "flat" },
|
|
75
|
+
{ checkpointId: "r1_0", score: 0.78, source: "raptor", raptorLevel: 1 },
|
|
76
|
+
{ checkpointId: "chkpt_007", score: 0.65, source: "cross-repo" },
|
|
77
|
+
], sd);
|
|
78
|
+
const recalls = listTurnRecall(turnId, sd);
|
|
79
|
+
assert.equal(recalls.length, 3);
|
|
80
|
+
assert.equal(recalls[0].checkpointId, "chkpt_003", "sorted by score desc");
|
|
81
|
+
assert.equal(recalls[0].score, 0.91);
|
|
82
|
+
const raptorHit = recalls.find((r) => r.source === "raptor");
|
|
83
|
+
assert.ok(raptorHit, "raptor hit present");
|
|
84
|
+
assert.equal(raptorHit.raptorLevel, 1);
|
|
85
|
+
assert.equal(raptorHit.checkpointId, "r1_0");
|
|
86
|
+
// Re-record with a changed score → upsert, no duplicate.
|
|
87
|
+
recordTurnRecall(turnId, [{ checkpointId: "chkpt_003", score: 0.95, source: "flat" }], sd);
|
|
88
|
+
const recalls2 = listTurnRecall(turnId, sd);
|
|
89
|
+
assert.equal(recalls2.length, 3, "no duplicate on re-record");
|
|
90
|
+
const updated = recalls2.find((r) => r.checkpointId === "chkpt_003");
|
|
91
|
+
assert.equal(updated.score, 0.95, "score updated");
|
|
92
|
+
});
|
|
93
|
+
// ─── 3. listConversationTurns ───────────────────────────────────────────────
|
|
94
|
+
test("S43-3: listConversationTurns returns turns in order", () => {
|
|
95
|
+
const sd = stateDir();
|
|
96
|
+
openStore(sd);
|
|
97
|
+
const conv = newConversationId();
|
|
98
|
+
for (let i = 1; i <= 4; i++) {
|
|
99
|
+
recordTurn({
|
|
100
|
+
conversationId: conv,
|
|
101
|
+
sessionId: "sess-1",
|
|
102
|
+
turnIndex: i,
|
|
103
|
+
endedAt: i * 100,
|
|
104
|
+
}, sd);
|
|
105
|
+
}
|
|
106
|
+
const turns = listConversationTurns(conv, sd);
|
|
107
|
+
assert.equal(turns.length, 4);
|
|
108
|
+
assert.deepEqual(turns.map((t) => t.turnIndex), [1, 2, 3, 4]);
|
|
109
|
+
});
|
|
110
|
+
// ─── 4. ensureConversationId persists + is stable across resumes ───────────
|
|
111
|
+
test("S43-4: ensureConversationId generates once, persists, survives reload", () => {
|
|
112
|
+
const sd = stateDir();
|
|
113
|
+
openStore(sd);
|
|
114
|
+
const sid = "sess-resume";
|
|
115
|
+
// First call generates + persists.
|
|
116
|
+
const conv1 = ensureConversationId(sid, sd);
|
|
117
|
+
assert.ok(conv1.startsWith("conv_"), "generated id has prefix");
|
|
118
|
+
// Second call (simulated resume) returns the same id — reads session_state.
|
|
119
|
+
const conv2 = ensureConversationId(sid, sd);
|
|
120
|
+
assert.equal(conv2, conv1, "stable across resumes");
|
|
121
|
+
// A different session gets a different conversation id.
|
|
122
|
+
const conv3 = ensureConversationId("sess-other", sd);
|
|
123
|
+
assert.notEqual(conv3, conv1);
|
|
124
|
+
});
|
|
125
|
+
// ─── 5. forkConversation records lineage + returns recall set ──────────────
|
|
126
|
+
test("S43-5: forkConversation creates child + returns parent's recall set", () => {
|
|
127
|
+
const sd = stateDir();
|
|
128
|
+
openStore(sd);
|
|
129
|
+
const parent = newConversationId();
|
|
130
|
+
// Parent conversation: turn 1 with 2 recalled checkpoints.
|
|
131
|
+
const turnId = recordTurn({
|
|
132
|
+
conversationId: parent,
|
|
133
|
+
sessionId: "sess-parent",
|
|
134
|
+
turnIndex: 1,
|
|
135
|
+
}, sd);
|
|
136
|
+
recordTurnRecall(turnId, [
|
|
137
|
+
{ checkpointId: "chkpt_001", score: 0.9, source: "flat" },
|
|
138
|
+
{ checkpointId: "chkpt_002", score: 0.7, source: "flat" },
|
|
139
|
+
], sd);
|
|
140
|
+
// Fork at turn 1.
|
|
141
|
+
const { conversationId: child, recalled } = forkConversation(parent, turnId, sd);
|
|
142
|
+
assert.ok(child.startsWith("conv_"), "child is a conversation id");
|
|
143
|
+
assert.notEqual(child, parent, "child differs from parent");
|
|
144
|
+
assert.equal(recalled.length, 2, "parent's recall set returned for replay");
|
|
145
|
+
assert.ok(recalled.some((r) => r.checkpointId === "chkpt_001"), "includes parent's recalled checkpoint");
|
|
146
|
+
// The child conversation is recorded as a branch.
|
|
147
|
+
const branches = openStore(sd).prepare("SELECT parent_conversation_id, fork_turn_id FROM conversation_branches WHERE conversation_id = ?").get(child);
|
|
148
|
+
assert.equal(branches.parent_conversation_id, parent);
|
|
149
|
+
assert.equal(branches.fork_turn_id, turnId);
|
|
150
|
+
});
|
|
151
|
+
// ─── 6. clearTurns removes turns + their recall rows for a session ──────────
|
|
152
|
+
test("S43-6: clearTurns removes turns + cascade turn_recall", () => {
|
|
153
|
+
const sd = stateDir();
|
|
154
|
+
openStore(sd);
|
|
155
|
+
const conv = newConversationId();
|
|
156
|
+
const turnId = recordTurn({
|
|
157
|
+
conversationId: conv,
|
|
158
|
+
sessionId: "sess-clear",
|
|
159
|
+
turnIndex: 1,
|
|
160
|
+
}, sd);
|
|
161
|
+
recordTurnRecall(turnId, [{ checkpointId: "x", score: 0.5, source: "flat" }], sd);
|
|
162
|
+
assert.equal(listTurnRecall(turnId, sd).length, 1);
|
|
163
|
+
clearTurns("sess-clear", sd);
|
|
164
|
+
assert.equal(getTurnById(turnId, sd), null, "turn gone");
|
|
165
|
+
assert.equal(listTurnRecall(turnId, sd).length, 0, "recall rows cascaded");
|
|
166
|
+
});
|
|
167
|
+
// ─── 7. conversation id survives across SessionState save/load ──────────────
|
|
168
|
+
test("S43-7: conversationId round-trips through SessionState", () => {
|
|
169
|
+
const sd = stateDir();
|
|
170
|
+
openStore(sd);
|
|
171
|
+
const sid = "sess-rt";
|
|
172
|
+
// Save a state with a conversationId.
|
|
173
|
+
const conv = "conv_roundtrip";
|
|
174
|
+
saveSessionState(sid, {
|
|
175
|
+
injectedCheckpointIds: [],
|
|
176
|
+
storedRegionHashes: [],
|
|
177
|
+
conversationId: conv,
|
|
178
|
+
}, sd);
|
|
179
|
+
const loaded = loadSessionState(sid, sd);
|
|
180
|
+
assert.equal(loaded.conversationId, conv, "conversationId persisted");
|
|
181
|
+
// ensureConversationId picks up the existing one.
|
|
182
|
+
assert.equal(ensureConversationId(sid, sd), conv, "ensure keeps existing");
|
|
183
|
+
});
|
|
@@ -38,6 +38,17 @@ export function decodeEmbedding(buf) {
|
|
|
38
38
|
export function jsonText(v) {
|
|
39
39
|
return JSON.stringify(v ?? []);
|
|
40
40
|
}
|
|
41
|
+
/** Safe JSON parse — returns `fallback` on null/undefined/corrupt input. */
|
|
42
|
+
export function safeJson(s, fallback) {
|
|
43
|
+
if (!s)
|
|
44
|
+
return fallback;
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(s);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return fallback;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
41
52
|
// In-process cache so the same stateDir reuses one connection (and so a fresh
|
|
42
53
|
// VectorStore over the same dir shares the open DB). Cross-process durability
|
|
43
54
|
// comes from reopening the same file path — proven by the integration test.
|
|
@@ -50,7 +61,7 @@ export function openStore(stateDir = getStateDir()) {
|
|
|
50
61
|
// instead of closeStore) would surface as "database is not open" on the
|
|
51
62
|
// next reuse. Detect and evict so callers never see a dead handle.
|
|
52
63
|
try {
|
|
53
|
-
existing.
|
|
64
|
+
existing.exec("SELECT 1");
|
|
54
65
|
return existing;
|
|
55
66
|
}
|
|
56
67
|
catch {
|
|
@@ -100,9 +111,9 @@ export function rowToCheckpoint(row) {
|
|
|
100
111
|
summary: row.summary ?? "",
|
|
101
112
|
topicSummary: row.topic_summary ?? undefined,
|
|
102
113
|
summaryHash: row.summary_hash ?? undefined,
|
|
103
|
-
keyDecisions:
|
|
104
|
-
nextSteps:
|
|
105
|
-
filesModified:
|
|
114
|
+
keyDecisions: safeJson(row.key_decisions, []),
|
|
115
|
+
nextSteps: safeJson(row.next_steps, []),
|
|
116
|
+
filesModified: safeJson(row.files_modified, []),
|
|
106
117
|
tokenEstimate: row.token_estimate ?? 0,
|
|
107
118
|
originalTokenEstimate: row.original_token_estimate ?? undefined,
|
|
108
119
|
regionHash: row.region_hash ?? "",
|
package/dist/src/store/sqlite.js
CHANGED
package/dist/src/store.js
CHANGED
|
@@ -28,7 +28,7 @@ export function normalizeSessionId(sessionId) {
|
|
|
28
28
|
if (sessionId.startsWith("sess_"))
|
|
29
29
|
return sessionId;
|
|
30
30
|
if (sessionId.length >= 32 && sessionId.includes("-")) {
|
|
31
|
-
return `sess_${sessionId.replace(/-/g, "").slice(0, 16)}`;
|
|
31
|
+
return `sess_${sessionId.replace(/-/g, "").toLowerCase().slice(0, 16)}`;
|
|
32
32
|
}
|
|
33
33
|
return `sess_${sessionId}`;
|
|
34
34
|
}
|
|
@@ -64,7 +64,7 @@ export function writeGzJson(path, data) {
|
|
|
64
64
|
}
|
|
65
65
|
/** Append a checkpoint to the per-session checkpoint file (gzipped). */
|
|
66
66
|
export function appendCheckpoint(cp, stateDir = getStateDir()) {
|
|
67
|
-
const file = join(stateDir, `${cp.sessionId}.checkpoints.json.gz`);
|
|
67
|
+
const file = join(stateDir, `${normalizeSessionId(cp.sessionId)}.checkpoints.json.gz`);
|
|
68
68
|
const existing = readGzJson(file, []);
|
|
69
69
|
existing.push(cp);
|
|
70
70
|
writeGzJson(file, existing);
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-search-cache.test.ts — QA perf/correctness regression tests for the
|
|
3
|
+
* RAPTOR serve path and the per-session raptorCache.
|
|
4
|
+
*
|
|
5
|
+
* Guards two invariants the S42B/S25 wiring introduced:
|
|
6
|
+
* (a) Enabling RAPTOR (multilevel ON) never returns FEWER relevant hits than
|
|
7
|
+
* flat-only at the same k — no cache-hit loss / MMR starvation.
|
|
8
|
+
* (b) The per-search listCheckpoints is shared (not double-loaded): the
|
|
9
|
+
* warm-search latency at 200 checkpoints stays under the QA budget.
|
|
10
|
+
*
|
|
11
|
+
* No network. Real stores, temp state dirs.
|
|
12
|
+
*/
|
|
13
|
+
import { test, beforeEach, afterEach } from "node:test";
|
|
14
|
+
import assert from "node:assert/strict";
|
|
15
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { VectorStore, vectorList, vectorSearch } from "./vectorStore.js";
|
|
19
|
+
import { runRaptor } from "./dedup/raptor/index.js";
|
|
20
|
+
import { compactSession } from "./engine.js";
|
|
21
|
+
import { loadDedupConfig } from "./config/dedup.js";
|
|
22
|
+
import { normalizeSessionId } from "./store.js";
|
|
23
|
+
let tmpDir;
|
|
24
|
+
let counter = 0;
|
|
25
|
+
beforeEach(() => {
|
|
26
|
+
tmpDir = mkdtempSync(join(tmpdir(), "mc-vc-"));
|
|
27
|
+
});
|
|
28
|
+
afterEach(() => {
|
|
29
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
30
|
+
});
|
|
31
|
+
function stateDir() {
|
|
32
|
+
return join(tmpDir, `run-${counter++}`);
|
|
33
|
+
}
|
|
34
|
+
function cfg(overrides) {
|
|
35
|
+
return {
|
|
36
|
+
...loadDedupConfig(),
|
|
37
|
+
RAPTOR_ENABLED: true,
|
|
38
|
+
L0_ENABLED: false,
|
|
39
|
+
L1_ENABLED: false,
|
|
40
|
+
L2_ENABLED: true,
|
|
41
|
+
...overrides,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function msg(text, toolName) {
|
|
45
|
+
return toolName
|
|
46
|
+
? { role: "assistant", text, toolName, input: text, output: text }
|
|
47
|
+
: { role: "user", text };
|
|
48
|
+
}
|
|
49
|
+
function seedTwoTopics(store, sid, perTopic) {
|
|
50
|
+
const topics = [
|
|
51
|
+
"database connection pool postgres query optimizer",
|
|
52
|
+
"user interface button render react component",
|
|
53
|
+
];
|
|
54
|
+
for (let i = 1; i <= perTopic * 2; i++) {
|
|
55
|
+
const t = topics[i % 2];
|
|
56
|
+
compactSession({
|
|
57
|
+
sessionId: sid,
|
|
58
|
+
messages: [msg(`${t} checkpoint ${i} alpha beta gamma`), msg(`ack ${i}`, "Edit")],
|
|
59
|
+
keepFrom: 2,
|
|
60
|
+
timestamp: i,
|
|
61
|
+
}, store);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// ─── (a) RAPTOR ON never loses hits vs OFF ──────────────────────────────────
|
|
65
|
+
test("QA cache: RAPTOR-ON returns >= flat-only hits at same k (no MMR starvation)", () => {
|
|
66
|
+
const k = 5;
|
|
67
|
+
// Build the same session twice (isolated state dirs) so the comparison is
|
|
68
|
+
// exact: same checkpoints, same embeddings, only the RAPTOR flag differs.
|
|
69
|
+
const queries = ["database connection pool", "user interface button", "alpha beta gamma"];
|
|
70
|
+
for (const q of queries) {
|
|
71
|
+
let offHits = 0;
|
|
72
|
+
let onHits = 0;
|
|
73
|
+
for (const raptorOn of [false, true]) {
|
|
74
|
+
const sd = stateDir();
|
|
75
|
+
const s = new VectorStore({
|
|
76
|
+
dedupSim: 0.9,
|
|
77
|
+
stateDir: sd,
|
|
78
|
+
config: raptorOn
|
|
79
|
+
? cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: true })
|
|
80
|
+
: cfg({ RAPTOR_ENABLED: false }),
|
|
81
|
+
});
|
|
82
|
+
const sid = `cache-${q.length}-${raptorOn ? 1 : 0}`;
|
|
83
|
+
seedTwoTopics(s, sid, 8); // 16 checkpoints, 2 topics
|
|
84
|
+
if (raptorOn) {
|
|
85
|
+
const nsid = normalizeSessionId(sid);
|
|
86
|
+
runRaptor(vectorList(s, nsid).map((cp) => ({
|
|
87
|
+
id: cp.checkpointId,
|
|
88
|
+
messages: [],
|
|
89
|
+
sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
|
|
90
|
+
embedding: cp.embedding,
|
|
91
|
+
})), { stateDir: sd, sessionId: nsid });
|
|
92
|
+
}
|
|
93
|
+
process.env.RAPTOR_SHADOW_MODE = raptorOn ? "false" : "true";
|
|
94
|
+
const hits = vectorSearch(s, sid, q, k);
|
|
95
|
+
if (raptorOn)
|
|
96
|
+
onHits = hits.length;
|
|
97
|
+
else
|
|
98
|
+
offHits = hits.length;
|
|
99
|
+
}
|
|
100
|
+
assert.ok(onHits >= offHits, `RAPTOR ON (${onHits}) >= OFF (${offHits}) for query "${q}"`);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
// ─── (b) Warm-search latency budget at 200 checkpoints ─────────────────────
|
|
104
|
+
test("QA perf: warm RAPTOR search at 200 checkpoints stays under budget (no double scan)", () => {
|
|
105
|
+
const sd = stateDir();
|
|
106
|
+
const s = new VectorStore({
|
|
107
|
+
dedupSim: 0.9,
|
|
108
|
+
stateDir: sd,
|
|
109
|
+
config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true }),
|
|
110
|
+
});
|
|
111
|
+
const sid = "perf";
|
|
112
|
+
seedTwoTopics(s, sid, 100); // 200 checkpoints
|
|
113
|
+
const nsid = normalizeSessionId(sid);
|
|
114
|
+
runRaptor(vectorList(s, nsid).map((cp) => ({
|
|
115
|
+
id: cp.checkpointId,
|
|
116
|
+
messages: [],
|
|
117
|
+
sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
|
|
118
|
+
embedding: cp.embedding,
|
|
119
|
+
})), { stateDir: sd, sessionId: nsid });
|
|
120
|
+
process.env.RAPTOR_SHADOW_MODE = "false";
|
|
121
|
+
// Warm the cache + embedder.
|
|
122
|
+
vectorSearch(s, sid, "database connection pool", 5);
|
|
123
|
+
vectorSearch(s, sid, "database connection pool", 5);
|
|
124
|
+
// Time a burst of warm searches. The double-scan fix keeps this well under
|
|
125
|
+
// the pre-fix cost. Budget is generous (10ms/search) to avoid CI flake;
|
|
126
|
+
// the regression it guards is the ~2x blowup from listCheckpoints running
|
|
127
|
+
// twice per search.
|
|
128
|
+
const ITERS = 50;
|
|
129
|
+
const t0 = Date.now();
|
|
130
|
+
for (let i = 0; i < ITERS; i++)
|
|
131
|
+
vectorSearch(s, sid, "database connection pool", 5);
|
|
132
|
+
const perSearch = (Date.now() - t0) / ITERS;
|
|
133
|
+
assert.ok(perSearch < 10, `warm search ${perSearch.toFixed(2)}ms < 10ms budget (double-scan regression guard)`);
|
|
134
|
+
});
|
|
135
|
+
// ─── (c) No duplicate checkpointIds in merged results ──────────────────────
|
|
136
|
+
test("QA cache: RAPTOR merge produces no duplicate checkpointIds", () => {
|
|
137
|
+
const sd = stateDir();
|
|
138
|
+
const s = new VectorStore({
|
|
139
|
+
dedupSim: 0.9,
|
|
140
|
+
stateDir: sd,
|
|
141
|
+
config: cfg({ RAPTOR_MULTILEVEL_ENABLED: true, RAPTOR_LEAF_EXPANSION: false }),
|
|
142
|
+
});
|
|
143
|
+
const sid = "dup";
|
|
144
|
+
seedTwoTopics(s, sid, 8);
|
|
145
|
+
const nsid = normalizeSessionId(sid);
|
|
146
|
+
runRaptor(vectorList(s, nsid).map((cp) => ({
|
|
147
|
+
id: cp.checkpointId,
|
|
148
|
+
messages: [],
|
|
149
|
+
sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
|
|
150
|
+
embedding: cp.embedding,
|
|
151
|
+
})), { stateDir: sd, sessionId: nsid });
|
|
152
|
+
process.env.RAPTOR_SHADOW_MODE = "false";
|
|
153
|
+
const hits = vectorSearch(s, sid, "database connection pool", 8);
|
|
154
|
+
const ids = hits.map((h) => h.checkpoint.checkpointId);
|
|
155
|
+
const dupes = ids.filter((id, i) => ids.indexOf(id) !== i);
|
|
156
|
+
assert.equal(dupes.length, 0, "no duplicate checkpointIds in merged results");
|
|
157
|
+
});
|