pi-mega-compact 0.8.24 → 0.8.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/extensions/mega-compact-s38.test.js +263 -14
- package/dist/extensions/mega-compact.js +15 -0
- package/dist/extensions/mega-config.js +3 -0
- package/dist/extensions/mega-events/agent-handlers.js +211 -26
- package/dist/extensions/mega-events/context-handler.js +45 -7
- package/dist/extensions/mega-events/error-classifier.js +125 -18
- package/dist/extensions/mega-pipeline/compact.js +24 -13
- package/dist/extensions/mega-pipeline/recall.js +31 -2
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime.js +58 -5
- package/dist/src/boundary.js +79 -43
- package/dist/src/boundary.test.js +119 -2
- package/dist/src/canary.js +10 -0
- package/dist/src/config/dedup.js +14 -0
- package/dist/src/config.js +3 -1
- package/dist/src/dedup/raptor/buildHistory.js +164 -0
- package/dist/src/dedup/raptor/buildHistory.test.js +292 -0
- package/dist/src/dedup/raptor/index.js +38 -0
- package/dist/src/dedup/raptor/multilevel-serve.test.js +229 -0
- package/dist/src/dedup/raptor/multilevel.js +17 -5
- package/dist/src/dedup/raptor/multilevel.test.js +36 -1
- package/dist/src/dedup/raptor/raptor.test.js +43 -0
- package/dist/src/dedup/raptor/retrieval.js +14 -2
- package/dist/src/dedup/raptor/retrieval.test.js +95 -0
- package/dist/src/dedup/raptor/serve-gate.test.js +298 -0
- package/dist/src/dedup/raptor/summarizer.js +1 -0
- package/dist/src/dedup/raptor/tree.js +16 -2
- package/dist/src/engine.js +18 -2
- package/dist/src/httpEmbedder.js +96 -6
- package/dist/src/httpEmbedder.test.js +277 -0
- package/dist/src/mechanical-fix.test.js +65 -0
- package/dist/src/raptor-inject-summaries.test.js +162 -0
- package/dist/src/recall.js +153 -24
- package/dist/src/recall.test.js +179 -4
- package/dist/src/store/sqlite/dedup-mirror.js +32 -15
- package/dist/src/store/sqlite/maintenance.js +2 -2
- package/dist/src/store/sqlite/mechanical-fix.test.js +146 -0
- package/dist/src/store/sqlite/memories.js +5 -5
- package/dist/src/store/sqlite/meta.js +1 -1
- package/dist/src/store/sqlite/raptor.js +56 -17
- package/dist/src/store/sqlite/raptor.test.js +106 -0
- package/dist/src/store/sqlite/schema.js +90 -1
- package/dist/src/store/sqlite/session-state.js +9 -3
- package/dist/src/store/sqlite/stats.js +9 -5
- package/dist/src/store/sqlite/turns.js +179 -0
- package/dist/src/store/sqlite/turns.test.js +183 -0
- package/dist/src/store/sqlite/utils.js +15 -4
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/store.js +2 -2
- package/dist/src/vector-search-cache.test.js +157 -0
- package/dist/src/vector-search.js +107 -15
- package/dist/src/vectorStore.js +36 -8
- package/extensions/mega-compact-s38.test.ts +259 -14
- package/extensions/mega-compact.ts +15 -0
- package/extensions/mega-config.ts +18 -0
- package/extensions/mega-dashboard.ts +10 -1
- package/extensions/mega-events/agent-handlers.ts +211 -26
- package/extensions/mega-events/context-handler.ts +43 -7
- package/extensions/mega-events/error-classifier.ts +125 -17
- package/extensions/mega-pipeline/compact.ts +28 -16
- package/extensions/mega-pipeline/recall.ts +34 -2
- package/extensions/mega-runtime/dashboard-snapshot.ts +8 -0
- package/extensions/mega-runtime/helpers.ts +25 -1
- package/extensions/mega-runtime/runtime-snapshot.ts +4 -0
- package/extensions/mega-runtime/runtime.ts +69 -23
- package/package.json +1 -1
- package/src/boundary.test.ts +128 -2
- package/src/boundary.ts +75 -39
- package/src/canary.ts +10 -0
- package/src/config/dedup.ts +25 -0
- package/src/config.ts +3 -1
- package/src/dedup/raptor/buildHistory.test.ts +353 -0
- package/src/dedup/raptor/buildHistory.ts +259 -0
- package/src/dedup/raptor/index.ts +38 -0
- package/src/dedup/raptor/multilevel-serve.test.ts +273 -0
- package/src/dedup/raptor/multilevel.test.ts +47 -0
- package/src/dedup/raptor/multilevel.ts +18 -8
- package/src/dedup/raptor/raptor.test.ts +59 -0
- package/src/dedup/raptor/retrieval.test.ts +118 -0
- package/src/dedup/raptor/retrieval.ts +14 -2
- package/src/dedup/raptor/serve-gate.test.ts +348 -0
- package/src/dedup/raptor/summarizer.ts +1 -0
- package/src/dedup/raptor/tree.ts +17 -2
- package/src/engine.ts +32 -3
- package/src/httpEmbedder.test.ts +286 -0
- package/src/httpEmbedder.ts +98 -8
- package/src/mechanical-fix.test.ts +70 -0
- package/src/raptor-inject-summaries.test.ts +228 -0
- package/src/recall.test.ts +220 -4
- package/src/recall.ts +462 -265
- package/src/store/sqlite/dedup-mirror.ts +35 -18
- package/src/store/sqlite/maintenance.ts +2 -2
- package/src/store/sqlite/mechanical-fix.test.ts +162 -0
- package/src/store/sqlite/memories.ts +5 -5
- package/src/store/sqlite/meta.ts +1 -1
- package/src/store/sqlite/raptor.test.ts +139 -0
- package/src/store/sqlite/raptor.ts +135 -81
- package/src/store/sqlite/schema.ts +90 -1
- package/src/store/sqlite/session-state.ts +9 -3
- package/src/store/sqlite/stats.ts +10 -8
- package/src/store/sqlite/turns.test.ts +218 -0
- package/src/store/sqlite/turns.ts +302 -0
- package/src/store/sqlite/utils.ts +14 -4
- package/src/store/sqlite.ts +1 -0
- package/src/store.ts +9 -2
- package/src/vector-search-cache.test.ts +190 -0
- package/src/vector-search.ts +273 -156
- package/src/vectorStore.ts +443 -382
- package/extensions/mega-runtime/reset-runtime.ts +0 -80
|
@@ -2,106 +2,160 @@
|
|
|
2
2
|
* raptor.ts — Sprint 13 RAPTOR node persistence.
|
|
3
3
|
*/
|
|
4
4
|
import { getStateDir, normalizeSessionId } from "../../store.js";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
openStore,
|
|
7
|
+
withTx,
|
|
8
|
+
jsonText,
|
|
9
|
+
encodeEmbedding,
|
|
10
|
+
decodeEmbedding,
|
|
11
|
+
} from "./utils.js";
|
|
6
12
|
|
|
7
13
|
export interface StoredRaptorNode {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
id: string;
|
|
15
|
+
sessionId: string;
|
|
16
|
+
level: number;
|
|
17
|
+
parentId: string | null;
|
|
18
|
+
children: string[];
|
|
19
|
+
summary: string;
|
|
20
|
+
embedding: number[];
|
|
21
|
+
qualityMarker: string;
|
|
22
|
+
tokenEstimate: number;
|
|
23
|
+
/** S25: epoch ms when the tree containing this node was built. */
|
|
24
|
+
builtAt: number;
|
|
19
25
|
}
|
|
20
26
|
|
|
21
27
|
/** Persist a single RAPTOR node (upsert by (session_id, id)). */
|
|
22
|
-
export function upsertRaptorNode(
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
export function upsertRaptorNode(
|
|
29
|
+
node: StoredRaptorNode,
|
|
30
|
+
stateDir: string = getStateDir(),
|
|
31
|
+
): void {
|
|
32
|
+
const db = openStore(stateDir);
|
|
33
|
+
db.prepare(
|
|
34
|
+
`INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
|
|
35
|
+
VALUES(@id, @session_id, @level, @parent_id, @children, @summary, @embedding_blob, @quality_marker, @token_estimate, @built_at)
|
|
27
36
|
ON CONFLICT(session_id, id) DO UPDATE SET
|
|
28
37
|
level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
|
|
29
38
|
summary=excluded.summary, embedding_blob=excluded.embedding_blob,
|
|
30
39
|
quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
|
|
31
40
|
built_at=excluded.built_at`,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
).run({
|
|
42
|
+
id: node.id,
|
|
43
|
+
session_id: node.sessionId,
|
|
44
|
+
level: node.level,
|
|
45
|
+
parent_id: node.parentId,
|
|
46
|
+
children: jsonText(node.children),
|
|
47
|
+
summary: node.summary,
|
|
48
|
+
embedding_blob: encodeEmbedding(node.embedding),
|
|
49
|
+
quality_marker: node.qualityMarker,
|
|
50
|
+
token_estimate: node.tokenEstimate,
|
|
51
|
+
built_at: node.builtAt,
|
|
52
|
+
});
|
|
44
53
|
}
|
|
45
54
|
|
|
46
55
|
/** Persist an entire built RAPTOR tree for a session (shadow or live). */
|
|
47
56
|
export function saveRaptorTree(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
sessionId: string,
|
|
58
|
+
tree: {
|
|
59
|
+
nodes: Map<
|
|
60
|
+
string,
|
|
61
|
+
{
|
|
62
|
+
id: string;
|
|
63
|
+
level: number;
|
|
64
|
+
parentId: string | null;
|
|
65
|
+
children: string[];
|
|
66
|
+
summary: string;
|
|
67
|
+
embedding: number[];
|
|
68
|
+
qualityMarker: string;
|
|
69
|
+
tokenEstimate: number;
|
|
70
|
+
}
|
|
71
|
+
>;
|
|
72
|
+
},
|
|
73
|
+
builtAt: number,
|
|
74
|
+
stateDir: string = getStateDir(),
|
|
63
75
|
): void {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
76
|
+
const nsid = normalizeSessionId(sessionId);
|
|
77
|
+
const db = openStore(stateDir);
|
|
78
|
+
withTx(db, () => {
|
|
79
|
+
db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(nsid);
|
|
80
|
+
for (const node of tree.nodes.values()) {
|
|
81
|
+
upsertRaptorNode(
|
|
82
|
+
{
|
|
83
|
+
id: node.id,
|
|
84
|
+
sessionId: nsid,
|
|
85
|
+
level: node.level,
|
|
86
|
+
parentId: node.parentId,
|
|
87
|
+
children: node.children,
|
|
88
|
+
summary: node.summary,
|
|
89
|
+
embedding: node.embedding,
|
|
90
|
+
qualityMarker: node.qualityMarker,
|
|
91
|
+
tokenEstimate: node.tokenEstimate,
|
|
92
|
+
builtAt,
|
|
93
|
+
},
|
|
94
|
+
stateDir,
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Safe JSON array parse — returns [] on corrupt input. */
|
|
101
|
+
function safeJsonArray(raw: unknown): string[] {
|
|
102
|
+
if (typeof raw !== "string" || !raw) return [];
|
|
103
|
+
try { return JSON.parse(raw) as string[]; }
|
|
104
|
+
catch { return []; }
|
|
81
105
|
}
|
|
82
106
|
|
|
83
107
|
/** Load all RAPTOR nodes for a session. */
|
|
84
|
-
export function listRaptorNodes(
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
108
|
+
export function listRaptorNodes(
|
|
109
|
+
sessionId: string,
|
|
110
|
+
stateDir: string = getStateDir(),
|
|
111
|
+
): StoredRaptorNode[] {
|
|
112
|
+
const db = openStore(stateDir);
|
|
113
|
+
const rows = db
|
|
114
|
+
.prepare(
|
|
115
|
+
"SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC",
|
|
116
|
+
)
|
|
117
|
+
.all(normalizeSessionId(sessionId)) as any[];
|
|
118
|
+
return rows.map((row) => ({
|
|
119
|
+
id: row.id,
|
|
120
|
+
sessionId: row.session_id,
|
|
121
|
+
level: row.level,
|
|
122
|
+
parentId: row.parent_id ?? null,
|
|
123
|
+
children: safeJsonArray(row.children),
|
|
124
|
+
summary: row.summary ?? "",
|
|
125
|
+
embedding: decodeEmbedding(row.embedding_blob),
|
|
126
|
+
qualityMarker: row.quality_marker ?? "low",
|
|
127
|
+
tokenEstimate: row.token_estimate ?? 0,
|
|
128
|
+
builtAt: Number(row.built_at ?? 0),
|
|
129
|
+
}));
|
|
101
130
|
}
|
|
102
131
|
|
|
103
132
|
/** Delete all RAPTOR nodes for a session (rollback/cleanup). */
|
|
104
|
-
export function clearRaptorNodes(
|
|
105
|
-
|
|
106
|
-
|
|
133
|
+
export function clearRaptorNodes(
|
|
134
|
+
sessionId: string,
|
|
135
|
+
stateDir: string = getStateDir(),
|
|
136
|
+
): void {
|
|
137
|
+
const db = openStore(stateDir);
|
|
138
|
+
db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(
|
|
139
|
+
normalizeSessionId(sessionId),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* S25: newest built_at for a session's RAPTOR tree (0 if no nodes exist).
|
|
145
|
+
* Cheap indexed MAX query used by raptorSearchHits to validate cache freshness
|
|
146
|
+
* without rehydrating the full node Map.
|
|
147
|
+
*/
|
|
148
|
+
export function maxRaptorNodeBuiltAt(
|
|
149
|
+
sessionId: string,
|
|
150
|
+
stateDir: string = getStateDir(),
|
|
151
|
+
): number {
|
|
152
|
+
const db = openStore(stateDir);
|
|
153
|
+
const row = db
|
|
154
|
+
.prepare(
|
|
155
|
+
`SELECT MAX(built_at) AS max_built FROM raptor_nodes WHERE session_id = ?`,
|
|
156
|
+
)
|
|
157
|
+
.get(normalizeSessionId(sessionId)) as
|
|
158
|
+
| { max_built: number | null }
|
|
159
|
+
| undefined;
|
|
160
|
+
return row?.max_built ?? 0;
|
|
107
161
|
}
|
|
@@ -48,6 +48,11 @@ export function initSchema(db: DatabaseSync): void {
|
|
|
48
48
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_pk
|
|
49
49
|
ON context_chunks(session_id, id);
|
|
50
50
|
CREATE INDEX IF NOT EXISTS idx_chunks_session ON context_chunks(session_id);
|
|
51
|
+
-- S42D/QA perf: session-scoped timestamp ordering so the S25 RAPTOR
|
|
52
|
+
-- freshness guard's MAX(timestamp) (run on every RAPTOR search) is
|
|
53
|
+
-- index-satisfied rather than a full partition scan as checkpoints grow.
|
|
54
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_session_ts
|
|
55
|
+
ON context_chunks(session_id, timestamp DESC);
|
|
51
56
|
CREATE INDEX IF NOT EXISTS idx_chunks_region ON context_chunks(region_hash);
|
|
52
57
|
CREATE INDEX IF NOT EXISTS idx_chunks_content ON context_chunks(content_hash);
|
|
53
58
|
-- Partial UNIQUE (QA #1): null content_hash rows never violate the constraint;
|
|
@@ -77,7 +82,9 @@ export function initSchema(db: DatabaseSync): void {
|
|
|
77
82
|
CREATE TABLE IF NOT EXISTS session_state (
|
|
78
83
|
session_id TEXT PRIMARY KEY,
|
|
79
84
|
injected_checkpoint_ids TEXT, -- JSON array
|
|
80
|
-
stored_region_hashes TEXT
|
|
85
|
+
stored_region_hashes TEXT, -- JSON array
|
|
86
|
+
conversation_id TEXT, -- S43: groups turns across resumes (/clear → new root)
|
|
87
|
+
last_turn_id INTEGER -- S43: most recent turn id (stable fork refs)
|
|
81
88
|
);
|
|
82
89
|
|
|
83
90
|
CREATE TABLE IF NOT EXISTS meta (
|
|
@@ -104,6 +111,28 @@ export function initSchema(db: DatabaseSync): void {
|
|
|
104
111
|
);
|
|
105
112
|
CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
|
|
106
113
|
CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
|
|
114
|
+
-- S42D/QA perf: session-scoped built_at ordering so the per-search cache
|
|
115
|
+
-- invalidation MAX(built_at) is index-satisfied.
|
|
116
|
+
CREATE INDEX IF NOT EXISTS idx_raptor_session_built
|
|
117
|
+
ON raptor_nodes(session_id, built_at DESC);
|
|
118
|
+
|
|
119
|
+
-- S42D: structured RAPTOR build history. One row per tree build; the
|
|
120
|
+
-- freshness check (buildHistory.ts) uses completed_at + leaf_count to skip
|
|
121
|
+
-- unnecessary rebuilds when the tree is recent and the chunk count is stable.
|
|
122
|
+
CREATE TABLE IF NOT EXISTS raptor_build_history (
|
|
123
|
+
build_id TEXT PRIMARY KEY,
|
|
124
|
+
session_id TEXT NOT NULL,
|
|
125
|
+
state_dir TEXT NOT NULL,
|
|
126
|
+
started_at INTEGER NOT NULL,
|
|
127
|
+
completed_at INTEGER NOT NULL,
|
|
128
|
+
node_count INTEGER NOT NULL,
|
|
129
|
+
leaf_count INTEGER NOT NULL,
|
|
130
|
+
depth INTEGER NOT NULL,
|
|
131
|
+
config_json TEXT NOT NULL, -- serialized BuildOptions
|
|
132
|
+
coherence_score REAL, -- avg intra-cluster cosine (post-build)
|
|
133
|
+
timed_out INTEGER NOT NULL DEFAULT 0
|
|
134
|
+
);
|
|
135
|
+
CREATE INDEX IF NOT EXISTS idx_raptor_build_session ON raptor_build_history(session_id);
|
|
107
136
|
|
|
108
137
|
-- Foundation for future features (resume sessions, daily log, lessons
|
|
109
138
|
-- learned). Scaffolded now so all store data lives in SQLite from day one;
|
|
@@ -222,6 +251,59 @@ export function initSchema(db: DatabaseSync): void {
|
|
|
222
251
|
);
|
|
223
252
|
CREATE INDEX IF NOT EXISTS idx_epoch_session ON checkpoint_epochs(session_id, created_at DESC);
|
|
224
253
|
|
|
254
|
+
-- S43 (per-turn vector tracking): the relational spine for per-turn +
|
|
255
|
+
-- per-conversation memories. One row per turn_end; links turns to the
|
|
256
|
+
-- epoch that compacted them (epoch_id) and to the checkpoints/cluster
|
|
257
|
+
-- summaries that were RECALLED during that turn (turn_recall).
|
|
258
|
+
-- conversation_id groups turns across pi session resumes (/clear starts a
|
|
259
|
+
-- new conversation root; a fork carries parent_conversation_id).
|
|
260
|
+
CREATE TABLE IF NOT EXISTS turns (
|
|
261
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, -- global turn id
|
|
262
|
+
conversation_id TEXT NOT NULL,
|
|
263
|
+
session_id TEXT NOT NULL,
|
|
264
|
+
turn_index INTEGER NOT NULL, -- per-session turn (event.turnIndex)
|
|
265
|
+
role TEXT, -- 'user' | 'assistant' | 'tool' (turn's last role)
|
|
266
|
+
started_at INTEGER NOT NULL,
|
|
267
|
+
ended_at INTEGER, -- set at turn_end
|
|
268
|
+
ctx_tokens INTEGER, -- runtime.lastCtxTokens snapshot
|
|
269
|
+
ctx_percent REAL, -- runtime.lastCtxPercent
|
|
270
|
+
pressure_band TEXT, -- 'low'|'mid'|'high'|'critical'
|
|
271
|
+
model_id TEXT,
|
|
272
|
+
epoch_id TEXT, -- FK checkpoint_epochs (set when a compact closes this turn's epoch)
|
|
273
|
+
UNIQUE(session_id, turn_index)
|
|
274
|
+
);
|
|
275
|
+
CREATE INDEX IF NOT EXISTS idx_turns_conv ON turns(conversation_id, turn_index);
|
|
276
|
+
CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id, turn_index);
|
|
277
|
+
CREATE INDEX IF NOT EXISTS idx_turns_epoch ON turns(epoch_id) WHERE epoch_id IS NOT NULL;
|
|
278
|
+
|
|
279
|
+
-- S43: recall provenance — which checkpoints/cluster summaries were
|
|
280
|
+
-- injected at which turn, their score, and the path that sourced them.
|
|
281
|
+
-- This is the per-turn vector data that makes memory quality measurable
|
|
282
|
+
-- per turn and enables recall-to-point (replay these checkpoint_ids into a
|
|
283
|
+
-- forked session). source: 'flat' | 'raptor' | 'cross-repo' | 'memory'.
|
|
284
|
+
CREATE TABLE IF NOT EXISTS turn_recall (
|
|
285
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
286
|
+
turn_id INTEGER NOT NULL,
|
|
287
|
+
checkpoint_id TEXT NOT NULL,
|
|
288
|
+
score REAL NOT NULL,
|
|
289
|
+
source TEXT NOT NULL,
|
|
290
|
+
raptor_level INTEGER, -- set for RAPTOR cluster hits
|
|
291
|
+
UNIQUE(turn_id, checkpoint_id)
|
|
292
|
+
);
|
|
293
|
+
CREATE INDEX IF NOT EXISTS idx_turn_recall_turn ON turn_recall(turn_id);
|
|
294
|
+
CREATE INDEX IF NOT EXISTS idx_turn_recall_cp ON turn_recall(checkpoint_id);
|
|
295
|
+
|
|
296
|
+
-- S43: conversation branch/fork registry. A row per fork: the child
|
|
297
|
+
-- conversation inherits the parent's recall state at fork_turn_id as its
|
|
298
|
+
-- starting injected-set. The root conversation has no row here.
|
|
299
|
+
CREATE TABLE IF NOT EXISTS conversation_branches (
|
|
300
|
+
conversation_id TEXT PRIMARY KEY,
|
|
301
|
+
parent_conversation_id TEXT NOT NULL,
|
|
302
|
+
fork_turn_id INTEGER NOT NULL, -- FK turns.id at the branch point
|
|
303
|
+
created_at INTEGER NOT NULL
|
|
304
|
+
);
|
|
305
|
+
CREATE INDEX IF NOT EXISTS idx_conv_branch_parent ON conversation_branches(parent_conversation_id);
|
|
306
|
+
|
|
225
307
|
-- S27 Task 6: dedup_mirror for space-efficient deduplicated storage.
|
|
226
308
|
-- Each unique content_hash stores its bytes ONCE; raw_transcript rows
|
|
227
309
|
-- reference this table via content_ref instead of storing duplicate content_bytes inline.
|
|
@@ -299,6 +381,13 @@ export function initSchema(db: DatabaseSync): void {
|
|
|
299
381
|
// S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
|
|
300
382
|
// treated as stale → flat fallback (safe).
|
|
301
383
|
ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
|
|
384
|
+
// S43: turn_index on raw_transcript so a message points directly at its
|
|
385
|
+
// conversation turn (otherwise it must be inferred from seq ordering). NULL
|
|
386
|
+
// for legacy rows — turns written before S43 have no turn link.
|
|
387
|
+
ensureColumn(db, "raw_transcript", "turn_index", "INTEGER");
|
|
388
|
+
// S43: conversation_id + last_turn_id on session_state (legacy DBs have NULL).
|
|
389
|
+
ensureColumn(db, "session_state", "conversation_id", "TEXT");
|
|
390
|
+
ensureColumn(db, "session_state", "last_turn_id", "INTEGER");
|
|
302
391
|
// S35: idempotent seed of the 9 achievement rows. ON CONFLICT(id) DO
|
|
303
392
|
// NOTHING so a re-open never clobbers an already-unlocked row's
|
|
304
393
|
// unlocked_at. No user input reaches this SQL (PREVENT-002 safe).
|
|
@@ -14,6 +14,8 @@ function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
|
|
|
14
14
|
return {
|
|
15
15
|
injectedCheckpointIds: row.injected_checkpoint_ids ? JSON.parse(row.injected_checkpoint_ids) : [],
|
|
16
16
|
storedRegionHashes: row.stored_region_hashes ? JSON.parse(row.stored_region_hashes) : [],
|
|
17
|
+
conversationId: (row.conversation_id as string | null) ?? undefined,
|
|
18
|
+
lastTurnId: (row.last_turn_id as number | null) ?? undefined,
|
|
17
19
|
};
|
|
18
20
|
}
|
|
19
21
|
|
|
@@ -25,14 +27,18 @@ export function saveSessionState(sessionId: string, state: SessionState, stateDi
|
|
|
25
27
|
const db = openStore(stateDir);
|
|
26
28
|
const sid = normalizeSessionId(sessionId);
|
|
27
29
|
db.prepare(
|
|
28
|
-
`INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes)
|
|
29
|
-
VALUES(@sid, @inj, @reg)
|
|
30
|
+
`INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes, conversation_id, last_turn_id)
|
|
31
|
+
VALUES(@sid, @inj, @reg, @conv, @tid)
|
|
30
32
|
ON CONFLICT(session_id) DO UPDATE SET
|
|
31
33
|
injected_checkpoint_ids=excluded.injected_checkpoint_ids,
|
|
32
|
-
stored_region_hashes=excluded.stored_region_hashes
|
|
34
|
+
stored_region_hashes=excluded.stored_region_hashes,
|
|
35
|
+
conversation_id=excluded.conversation_id,
|
|
36
|
+
last_turn_id=excluded.last_turn_id`,
|
|
33
37
|
).run({
|
|
34
38
|
sid,
|
|
35
39
|
inj: jsonText(state.injectedCheckpointIds),
|
|
36
40
|
reg: jsonText(state.storedRegionHashes),
|
|
41
|
+
conv: state.conversationId ?? null,
|
|
42
|
+
tid: state.lastTurnId ?? null,
|
|
37
43
|
});
|
|
38
44
|
}
|
|
@@ -18,21 +18,23 @@ export function storeStats(sessionId: string, stateDir: string = getStateDir()):
|
|
|
18
18
|
const row = db
|
|
19
19
|
.prepare(
|
|
20
20
|
`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
|
|
21
|
-
MAX(id) AS
|
|
22
|
-
FROM context_chunks WHERE session_id =
|
|
21
|
+
MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS lastNum
|
|
22
|
+
FROM context_chunks WHERE session_id = ? AND dedup_status != 'removed'`,
|
|
23
23
|
)
|
|
24
|
-
.get(sid) as { c: number; tok: number;
|
|
24
|
+
.get(sid) as { c: number; tok: number; lastNum: number | null };
|
|
25
|
+
let lastCheckpointId: string | undefined;
|
|
25
26
|
let lastSummary: string | undefined;
|
|
26
|
-
if (row.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
if (row.lastNum != null) {
|
|
28
|
+
lastCheckpointId = `chkpt_${String(row.lastNum).padStart(3, "0")}`;
|
|
29
|
+
const s = db
|
|
30
|
+
.prepare("SELECT summary FROM context_chunks WHERE session_id = ? AND id = ?")
|
|
31
|
+
.get(sid, lastCheckpointId) as { summary: string } | undefined;
|
|
30
32
|
lastSummary = s?.summary;
|
|
31
33
|
}
|
|
32
34
|
return {
|
|
33
35
|
checkpointCount: row.c,
|
|
34
36
|
totalTokenEstimate: row.tok,
|
|
35
|
-
lastCheckpointId
|
|
37
|
+
lastCheckpointId,
|
|
36
38
|
lastSummary,
|
|
37
39
|
};
|
|
38
40
|
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turns.test.ts — S43 per-turn + conversation tracking tests.
|
|
3
|
+
*
|
|
4
|
+
* No network. Real stores with temp state dirs.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { test, beforeEach, afterEach } from "node:test";
|
|
8
|
+
import assert from "node:assert/strict";
|
|
9
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { openStore } from "./utils.js";
|
|
13
|
+
import {
|
|
14
|
+
recordTurn,
|
|
15
|
+
recordTurnRecall,
|
|
16
|
+
getTurn,
|
|
17
|
+
getTurnById,
|
|
18
|
+
listTurnRecall,
|
|
19
|
+
listConversationTurns,
|
|
20
|
+
ensureConversationId,
|
|
21
|
+
forkConversation,
|
|
22
|
+
newConversationId,
|
|
23
|
+
clearTurns,
|
|
24
|
+
} from "./turns.js";
|
|
25
|
+
import { loadSessionState, saveSessionState } from "./session-state.js";
|
|
26
|
+
|
|
27
|
+
let tmpDir: string;
|
|
28
|
+
let counter = 0;
|
|
29
|
+
|
|
30
|
+
beforeEach(() => {
|
|
31
|
+
tmpDir = mkdtempSync(join(tmpdir(), "mc-turns-"));
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
afterEach(() => {
|
|
35
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
function stateDir(): string {
|
|
39
|
+
return join(tmpDir, `run-${counter++}`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// ─── 1. recordTurn upserts + getTurn round-trip ────────────────────────────
|
|
43
|
+
|
|
44
|
+
test("S43-1: recordTurn inserts/updates + getTurn finds by conv+index", () => {
|
|
45
|
+
const sd = stateDir();
|
|
46
|
+
openStore(sd);
|
|
47
|
+
const conv = newConversationId();
|
|
48
|
+
const id = recordTurn({
|
|
49
|
+
conversationId: conv,
|
|
50
|
+
sessionId: "sess-1",
|
|
51
|
+
turnIndex: 3,
|
|
52
|
+
endedAt: 1000,
|
|
53
|
+
ctxTokens: 12000,
|
|
54
|
+
ctxPercent: 42.5,
|
|
55
|
+
pressureBand: "mid",
|
|
56
|
+
modelId: "claude-fable-5",
|
|
57
|
+
}, sd);
|
|
58
|
+
assert.ok(id > 0, "turn id returned");
|
|
59
|
+
const t = getTurn(conv, 3, sd);
|
|
60
|
+
assert.ok(t, "turn found");
|
|
61
|
+
assert.equal(t!.conversationId, conv);
|
|
62
|
+
assert.equal(t!.turnIndex, 3);
|
|
63
|
+
assert.equal(t!.ctxTokens, 12000);
|
|
64
|
+
assert.equal(t!.ctxPercent, 42.5);
|
|
65
|
+
assert.equal(t!.pressureBand, "mid");
|
|
66
|
+
assert.equal(t!.modelId, "claude-fable-5");
|
|
67
|
+
assert.equal(t!.endedAt, 1000);
|
|
68
|
+
// Re-record (upsert) with additional metrics — COALESCE preserves started_at.
|
|
69
|
+
const id2 = recordTurn({
|
|
70
|
+
conversationId: conv,
|
|
71
|
+
sessionId: "sess-1",
|
|
72
|
+
turnIndex: 3,
|
|
73
|
+
startedAt: 900,
|
|
74
|
+
ctxTokens: 12500,
|
|
75
|
+
}, sd);
|
|
76
|
+
assert.equal(id2, id, "same turn id on upsert");
|
|
77
|
+
const t2 = getTurn(conv, 3, sd);
|
|
78
|
+
assert.equal(t2!.ctxTokens, 12500, "metrics updated");
|
|
79
|
+
assert.equal(t2!.startedAt, 900, "started_at set on second write");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
// ─── 2. recordTurnRecall + listTurnRecall ───────────────────────────────────
|
|
83
|
+
|
|
84
|
+
test("S43-2: recordTurnRecall stores provenance + listTurnRecall returns it", () => {
|
|
85
|
+
const sd = stateDir();
|
|
86
|
+
openStore(sd);
|
|
87
|
+
const conv = newConversationId();
|
|
88
|
+
const turnId = recordTurn({
|
|
89
|
+
conversationId: conv,
|
|
90
|
+
sessionId: "sess-1",
|
|
91
|
+
turnIndex: 1,
|
|
92
|
+
}, sd);
|
|
93
|
+
recordTurnRecall(turnId, [
|
|
94
|
+
{ checkpointId: "chkpt_003", score: 0.91, source: "flat" },
|
|
95
|
+
{ checkpointId: "r1_0", score: 0.78, source: "raptor", raptorLevel: 1 },
|
|
96
|
+
{ checkpointId: "chkpt_007", score: 0.65, source: "cross-repo" },
|
|
97
|
+
], sd);
|
|
98
|
+
const recalls = listTurnRecall(turnId, sd);
|
|
99
|
+
assert.equal(recalls.length, 3);
|
|
100
|
+
assert.equal(recalls[0].checkpointId, "chkpt_003", "sorted by score desc");
|
|
101
|
+
assert.equal(recalls[0].score, 0.91);
|
|
102
|
+
const raptorHit = recalls.find((r) => r.source === "raptor");
|
|
103
|
+
assert.ok(raptorHit, "raptor hit present");
|
|
104
|
+
assert.equal(raptorHit!.raptorLevel, 1);
|
|
105
|
+
assert.equal(raptorHit!.checkpointId, "r1_0");
|
|
106
|
+
// Re-record with a changed score → upsert, no duplicate.
|
|
107
|
+
recordTurnRecall(turnId, [{ checkpointId: "chkpt_003", score: 0.95, source: "flat" }], sd);
|
|
108
|
+
const recalls2 = listTurnRecall(turnId, sd);
|
|
109
|
+
assert.equal(recalls2.length, 3, "no duplicate on re-record");
|
|
110
|
+
const updated = recalls2.find((r) => r.checkpointId === "chkpt_003");
|
|
111
|
+
assert.equal(updated!.score, 0.95, "score updated");
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// ─── 3. listConversationTurns ───────────────────────────────────────────────
|
|
115
|
+
|
|
116
|
+
test("S43-3: listConversationTurns returns turns in order", () => {
|
|
117
|
+
const sd = stateDir();
|
|
118
|
+
openStore(sd);
|
|
119
|
+
const conv = newConversationId();
|
|
120
|
+
for (let i = 1; i <= 4; i++) {
|
|
121
|
+
recordTurn({
|
|
122
|
+
conversationId: conv,
|
|
123
|
+
sessionId: "sess-1",
|
|
124
|
+
turnIndex: i,
|
|
125
|
+
endedAt: i * 100,
|
|
126
|
+
}, sd);
|
|
127
|
+
}
|
|
128
|
+
const turns = listConversationTurns(conv, sd);
|
|
129
|
+
assert.equal(turns.length, 4);
|
|
130
|
+
assert.deepEqual(turns.map((t) => t.turnIndex), [1, 2, 3, 4]);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// ─── 4. ensureConversationId persists + is stable across resumes ───────────
|
|
134
|
+
|
|
135
|
+
test("S43-4: ensureConversationId generates once, persists, survives reload", () => {
|
|
136
|
+
const sd = stateDir();
|
|
137
|
+
openStore(sd);
|
|
138
|
+
const sid = "sess-resume";
|
|
139
|
+
// First call generates + persists.
|
|
140
|
+
const conv1 = ensureConversationId(sid, sd);
|
|
141
|
+
assert.ok(conv1.startsWith("conv_"), "generated id has prefix");
|
|
142
|
+
// Second call (simulated resume) returns the same id — reads session_state.
|
|
143
|
+
const conv2 = ensureConversationId(sid, sd);
|
|
144
|
+
assert.equal(conv2, conv1, "stable across resumes");
|
|
145
|
+
// A different session gets a different conversation id.
|
|
146
|
+
const conv3 = ensureConversationId("sess-other", sd);
|
|
147
|
+
assert.notEqual(conv3, conv1);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
// ─── 5. forkConversation records lineage + returns recall set ──────────────
|
|
151
|
+
|
|
152
|
+
test("S43-5: forkConversation creates child + returns parent's recall set", () => {
|
|
153
|
+
const sd = stateDir();
|
|
154
|
+
openStore(sd);
|
|
155
|
+
const parent = newConversationId();
|
|
156
|
+
// Parent conversation: turn 1 with 2 recalled checkpoints.
|
|
157
|
+
const turnId = recordTurn({
|
|
158
|
+
conversationId: parent,
|
|
159
|
+
sessionId: "sess-parent",
|
|
160
|
+
turnIndex: 1,
|
|
161
|
+
}, sd);
|
|
162
|
+
recordTurnRecall(turnId, [
|
|
163
|
+
{ checkpointId: "chkpt_001", score: 0.9, source: "flat" },
|
|
164
|
+
{ checkpointId: "chkpt_002", score: 0.7, source: "flat" },
|
|
165
|
+
], sd);
|
|
166
|
+
// Fork at turn 1.
|
|
167
|
+
const { conversationId: child, recalled } = forkConversation(parent, turnId, sd);
|
|
168
|
+
assert.ok(child.startsWith("conv_"), "child is a conversation id");
|
|
169
|
+
assert.notEqual(child, parent, "child differs from parent");
|
|
170
|
+
assert.equal(recalled.length, 2, "parent's recall set returned for replay");
|
|
171
|
+
assert.ok(
|
|
172
|
+
recalled.some((r) => r.checkpointId === "chkpt_001"),
|
|
173
|
+
"includes parent's recalled checkpoint",
|
|
174
|
+
);
|
|
175
|
+
// The child conversation is recorded as a branch.
|
|
176
|
+
const branches = openStore(sd).prepare(
|
|
177
|
+
"SELECT parent_conversation_id, fork_turn_id FROM conversation_branches WHERE conversation_id = ?",
|
|
178
|
+
).get(child) as { parent_conversation_id: string; fork_turn_id: number };
|
|
179
|
+
assert.equal(branches.parent_conversation_id, parent);
|
|
180
|
+
assert.equal(branches.fork_turn_id, turnId);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// ─── 6. clearTurns removes turns + their recall rows for a session ──────────
|
|
184
|
+
|
|
185
|
+
test("S43-6: clearTurns removes turns + cascade turn_recall", () => {
|
|
186
|
+
const sd = stateDir();
|
|
187
|
+
openStore(sd);
|
|
188
|
+
const conv = newConversationId();
|
|
189
|
+
const turnId = recordTurn({
|
|
190
|
+
conversationId: conv,
|
|
191
|
+
sessionId: "sess-clear",
|
|
192
|
+
turnIndex: 1,
|
|
193
|
+
}, sd);
|
|
194
|
+
recordTurnRecall(turnId, [{ checkpointId: "x", score: 0.5, source: "flat" }], sd);
|
|
195
|
+
assert.equal(listTurnRecall(turnId, sd).length, 1);
|
|
196
|
+
clearTurns("sess-clear", sd);
|
|
197
|
+
assert.equal(getTurnById(turnId, sd), null, "turn gone");
|
|
198
|
+
assert.equal(listTurnRecall(turnId, sd).length, 0, "recall rows cascaded");
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
// ─── 7. conversation id survives across SessionState save/load ──────────────
|
|
202
|
+
|
|
203
|
+
test("S43-7: conversationId round-trips through SessionState", () => {
|
|
204
|
+
const sd = stateDir();
|
|
205
|
+
openStore(sd);
|
|
206
|
+
const sid = "sess-rt";
|
|
207
|
+
// Save a state with a conversationId.
|
|
208
|
+
const conv = "conv_roundtrip";
|
|
209
|
+
saveSessionState(sid, {
|
|
210
|
+
injectedCheckpointIds: [],
|
|
211
|
+
storedRegionHashes: [],
|
|
212
|
+
conversationId: conv,
|
|
213
|
+
}, sd);
|
|
214
|
+
const loaded = loadSessionState(sid, sd);
|
|
215
|
+
assert.equal(loaded.conversationId, conv, "conversationId persisted");
|
|
216
|
+
// ensureConversationId picks up the existing one.
|
|
217
|
+
assert.equal(ensureConversationId(sid, sd), conv, "ensure keeps existing");
|
|
218
|
+
});
|