pi-mega-compact 0.8.24 → 0.8.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +26 -0
  2. package/dist/extensions/mega-compact-s38.test.js +263 -14
  3. package/dist/extensions/mega-compact.js +15 -0
  4. package/dist/extensions/mega-config.js +3 -0
  5. package/dist/extensions/mega-events/agent-handlers.js +211 -26
  6. package/dist/extensions/mega-events/context-handler.js +45 -7
  7. package/dist/extensions/mega-events/error-classifier.js +125 -18
  8. package/dist/extensions/mega-pipeline/compact.js +24 -13
  9. package/dist/extensions/mega-pipeline/recall.js +31 -2
  10. package/dist/extensions/mega-runtime/dashboard-snapshot.js +4 -0
  11. package/dist/extensions/mega-runtime/runtime-snapshot.js +4 -0
  12. package/dist/extensions/mega-runtime/runtime.js +58 -5
  13. package/dist/src/boundary.js +79 -43
  14. package/dist/src/boundary.test.js +119 -2
  15. package/dist/src/canary.js +10 -0
  16. package/dist/src/config/dedup.js +14 -0
  17. package/dist/src/config.js +3 -1
  18. package/dist/src/dedup/raptor/buildHistory.js +164 -0
  19. package/dist/src/dedup/raptor/buildHistory.test.js +292 -0
  20. package/dist/src/dedup/raptor/index.js +38 -0
  21. package/dist/src/dedup/raptor/multilevel-serve.test.js +229 -0
  22. package/dist/src/dedup/raptor/multilevel.js +17 -5
  23. package/dist/src/dedup/raptor/multilevel.test.js +36 -1
  24. package/dist/src/dedup/raptor/raptor.test.js +43 -0
  25. package/dist/src/dedup/raptor/retrieval.js +14 -2
  26. package/dist/src/dedup/raptor/retrieval.test.js +95 -0
  27. package/dist/src/dedup/raptor/serve-gate.test.js +298 -0
  28. package/dist/src/dedup/raptor/summarizer.js +1 -0
  29. package/dist/src/dedup/raptor/tree.js +16 -2
  30. package/dist/src/engine.js +18 -2
  31. package/dist/src/httpEmbedder.js +96 -6
  32. package/dist/src/httpEmbedder.test.js +277 -0
  33. package/dist/src/mechanical-fix.test.js +65 -0
  34. package/dist/src/raptor-inject-summaries.test.js +162 -0
  35. package/dist/src/recall.js +153 -24
  36. package/dist/src/recall.test.js +179 -4
  37. package/dist/src/store/sqlite/dedup-mirror.js +32 -15
  38. package/dist/src/store/sqlite/maintenance.js +2 -2
  39. package/dist/src/store/sqlite/mechanical-fix.test.js +146 -0
  40. package/dist/src/store/sqlite/memories.js +5 -5
  41. package/dist/src/store/sqlite/meta.js +1 -1
  42. package/dist/src/store/sqlite/raptor.js +56 -17
  43. package/dist/src/store/sqlite/raptor.test.js +106 -0
  44. package/dist/src/store/sqlite/schema.js +90 -1
  45. package/dist/src/store/sqlite/session-state.js +9 -3
  46. package/dist/src/store/sqlite/stats.js +9 -5
  47. package/dist/src/store/sqlite/turns.js +179 -0
  48. package/dist/src/store/sqlite/turns.test.js +183 -0
  49. package/dist/src/store/sqlite/utils.js +15 -4
  50. package/dist/src/store/sqlite.js +1 -0
  51. package/dist/src/store.js +2 -2
  52. package/dist/src/vector-search-cache.test.js +157 -0
  53. package/dist/src/vector-search.js +107 -15
  54. package/dist/src/vectorStore.js +36 -8
  55. package/extensions/mega-compact-s38.test.ts +259 -14
  56. package/extensions/mega-compact.ts +15 -0
  57. package/extensions/mega-config.ts +18 -0
  58. package/extensions/mega-dashboard.ts +10 -1
  59. package/extensions/mega-events/agent-handlers.ts +211 -26
  60. package/extensions/mega-events/context-handler.ts +43 -7
  61. package/extensions/mega-events/error-classifier.ts +125 -17
  62. package/extensions/mega-pipeline/compact.ts +28 -16
  63. package/extensions/mega-pipeline/recall.ts +34 -2
  64. package/extensions/mega-runtime/dashboard-snapshot.ts +8 -0
  65. package/extensions/mega-runtime/helpers.ts +25 -1
  66. package/extensions/mega-runtime/runtime-snapshot.ts +4 -0
  67. package/extensions/mega-runtime/runtime.ts +69 -23
  68. package/package.json +1 -1
  69. package/src/boundary.test.ts +128 -2
  70. package/src/boundary.ts +75 -39
  71. package/src/canary.ts +10 -0
  72. package/src/config/dedup.ts +25 -0
  73. package/src/config.ts +3 -1
  74. package/src/dedup/raptor/buildHistory.test.ts +353 -0
  75. package/src/dedup/raptor/buildHistory.ts +259 -0
  76. package/src/dedup/raptor/index.ts +38 -0
  77. package/src/dedup/raptor/multilevel-serve.test.ts +273 -0
  78. package/src/dedup/raptor/multilevel.test.ts +47 -0
  79. package/src/dedup/raptor/multilevel.ts +18 -8
  80. package/src/dedup/raptor/raptor.test.ts +59 -0
  81. package/src/dedup/raptor/retrieval.test.ts +118 -0
  82. package/src/dedup/raptor/retrieval.ts +14 -2
  83. package/src/dedup/raptor/serve-gate.test.ts +348 -0
  84. package/src/dedup/raptor/summarizer.ts +1 -0
  85. package/src/dedup/raptor/tree.ts +17 -2
  86. package/src/engine.ts +32 -3
  87. package/src/httpEmbedder.test.ts +286 -0
  88. package/src/httpEmbedder.ts +98 -8
  89. package/src/mechanical-fix.test.ts +70 -0
  90. package/src/raptor-inject-summaries.test.ts +228 -0
  91. package/src/recall.test.ts +220 -4
  92. package/src/recall.ts +462 -265
  93. package/src/store/sqlite/dedup-mirror.ts +35 -18
  94. package/src/store/sqlite/maintenance.ts +2 -2
  95. package/src/store/sqlite/mechanical-fix.test.ts +162 -0
  96. package/src/store/sqlite/memories.ts +5 -5
  97. package/src/store/sqlite/meta.ts +1 -1
  98. package/src/store/sqlite/raptor.test.ts +139 -0
  99. package/src/store/sqlite/raptor.ts +135 -81
  100. package/src/store/sqlite/schema.ts +90 -1
  101. package/src/store/sqlite/session-state.ts +9 -3
  102. package/src/store/sqlite/stats.ts +10 -8
  103. package/src/store/sqlite/turns.test.ts +218 -0
  104. package/src/store/sqlite/turns.ts +302 -0
  105. package/src/store/sqlite/utils.ts +14 -4
  106. package/src/store/sqlite.ts +1 -0
  107. package/src/store.ts +9 -2
  108. package/src/vector-search-cache.test.ts +190 -0
  109. package/src/vector-search.ts +273 -156
  110. package/src/vectorStore.ts +443 -382
  111. package/extensions/mega-runtime/reset-runtime.ts +0 -80
@@ -2,106 +2,160 @@
2
2
  * raptor.ts — Sprint 13 RAPTOR node persistence.
3
3
  */
4
4
  import { getStateDir, normalizeSessionId } from "../../store.js";
5
- import { openStore, jsonText, encodeEmbedding, decodeEmbedding } from "./utils.js";
5
+ import {
6
+ openStore,
7
+ withTx,
8
+ jsonText,
9
+ encodeEmbedding,
10
+ decodeEmbedding,
11
+ } from "./utils.js";
6
12
 
7
13
  export interface StoredRaptorNode {
8
- id: string;
9
- sessionId: string;
10
- level: number;
11
- parentId: string | null;
12
- children: string[];
13
- summary: string;
14
- embedding: number[];
15
- qualityMarker: string;
16
- tokenEstimate: number;
17
- /** S25: epoch ms when the tree containing this node was built. */
18
- builtAt: number;
14
+ id: string;
15
+ sessionId: string;
16
+ level: number;
17
+ parentId: string | null;
18
+ children: string[];
19
+ summary: string;
20
+ embedding: number[];
21
+ qualityMarker: string;
22
+ tokenEstimate: number;
23
+ /** S25: epoch ms when the tree containing this node was built. */
24
+ builtAt: number;
19
25
  }
20
26
 
21
27
  /** Persist a single RAPTOR node (upsert by (session_id, id)). */
22
- export function upsertRaptorNode(node: StoredRaptorNode, stateDir: string = getStateDir()): void {
23
- const db = openStore(stateDir);
24
- db.prepare(
25
- `INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
26
- VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
28
+ export function upsertRaptorNode(
29
+ node: StoredRaptorNode,
30
+ stateDir: string = getStateDir(),
31
+ ): void {
32
+ const db = openStore(stateDir);
33
+ db.prepare(
34
+ `INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
35
+ VALUES(@id, @session_id, @level, @parent_id, @children, @summary, @embedding_blob, @quality_marker, @token_estimate, @built_at)
27
36
  ON CONFLICT(session_id, id) DO UPDATE SET
28
37
  level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
29
38
  summary=excluded.summary, embedding_blob=excluded.embedding_blob,
30
39
  quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
31
40
  built_at=excluded.built_at`,
32
- ).run(
33
- node.id,
34
- node.sessionId,
35
- node.level,
36
- node.parentId,
37
- jsonText(node.children),
38
- node.summary,
39
- encodeEmbedding(node.embedding),
40
- node.qualityMarker,
41
- node.tokenEstimate,
42
- node.builtAt,
43
- );
41
+ ).run({
42
+ id: node.id,
43
+ session_id: node.sessionId,
44
+ level: node.level,
45
+ parent_id: node.parentId,
46
+ children: jsonText(node.children),
47
+ summary: node.summary,
48
+ embedding_blob: encodeEmbedding(node.embedding),
49
+ quality_marker: node.qualityMarker,
50
+ token_estimate: node.tokenEstimate,
51
+ built_at: node.builtAt,
52
+ });
44
53
  }
45
54
 
46
55
  /** Persist an entire built RAPTOR tree for a session (shadow or live). */
47
56
  export function saveRaptorTree(
48
- sessionId: string,
49
- tree: {
50
- nodes: Map<string, {
51
- id: string;
52
- level: number;
53
- parentId: string | null;
54
- children: string[];
55
- summary: string;
56
- embedding: number[];
57
- qualityMarker: string;
58
- tokenEstimate: number;
59
- }>
60
- },
61
- builtAt: number,
62
- stateDir: string = getStateDir(),
57
+ sessionId: string,
58
+ tree: {
59
+ nodes: Map<
60
+ string,
61
+ {
62
+ id: string;
63
+ level: number;
64
+ parentId: string | null;
65
+ children: string[];
66
+ summary: string;
67
+ embedding: number[];
68
+ qualityMarker: string;
69
+ tokenEstimate: number;
70
+ }
71
+ >;
72
+ },
73
+ builtAt: number,
74
+ stateDir: string = getStateDir(),
63
75
  ): void {
64
- for (const node of tree.nodes.values()) {
65
- upsertRaptorNode(
66
- {
67
- id: node.id,
68
- sessionId,
69
- level: node.level,
70
- parentId: node.parentId,
71
- children: node.children,
72
- summary: node.summary,
73
- embedding: node.embedding,
74
- qualityMarker: node.qualityMarker,
75
- tokenEstimate: node.tokenEstimate,
76
- builtAt,
77
- },
78
- stateDir,
79
- );
80
- }
76
+ const nsid = normalizeSessionId(sessionId);
77
+ const db = openStore(stateDir);
78
+ withTx(db, () => {
79
+ db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(nsid);
80
+ for (const node of tree.nodes.values()) {
81
+ upsertRaptorNode(
82
+ {
83
+ id: node.id,
84
+ sessionId: nsid,
85
+ level: node.level,
86
+ parentId: node.parentId,
87
+ children: node.children,
88
+ summary: node.summary,
89
+ embedding: node.embedding,
90
+ qualityMarker: node.qualityMarker,
91
+ tokenEstimate: node.tokenEstimate,
92
+ builtAt,
93
+ },
94
+ stateDir,
95
+ );
96
+ }
97
+ });
98
+ }
99
+
100
+ /** Safe JSON array parse — returns [] on corrupt input. */
101
+ function safeJsonArray(raw: unknown): string[] {
102
+ if (typeof raw !== "string" || !raw) return [];
103
+ try { return JSON.parse(raw) as string[]; }
104
+ catch { return []; }
81
105
  }
82
106
 
83
107
  /** Load all RAPTOR nodes for a session. */
84
- export function listRaptorNodes(sessionId: string, stateDir: string = getStateDir()): StoredRaptorNode[] {
85
- const db = openStore(stateDir);
86
- const rows = db
87
- .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
88
- .all(normalizeSessionId(sessionId)) as any[];
89
- return rows.map((row) => ({
90
- id: row.id,
91
- sessionId: row.session_id,
92
- level: row.level,
93
- parentId: row.parent_id ?? null,
94
- children: row.children ? JSON.parse(row.children) : [],
95
- summary: row.summary ?? "",
96
- embedding: decodeEmbedding(row.embedding_blob),
97
- qualityMarker: row.quality_marker ?? "low",
98
- tokenEstimate: row.token_estimate ?? 0,
99
- builtAt: Number(row.built_at ?? 0),
100
- }));
108
+ export function listRaptorNodes(
109
+ sessionId: string,
110
+ stateDir: string = getStateDir(),
111
+ ): StoredRaptorNode[] {
112
+ const db = openStore(stateDir);
113
+ const rows = db
114
+ .prepare(
115
+ "SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC",
116
+ )
117
+ .all(normalizeSessionId(sessionId)) as any[];
118
+ return rows.map((row) => ({
119
+ id: row.id,
120
+ sessionId: row.session_id,
121
+ level: row.level,
122
+ parentId: row.parent_id ?? null,
123
+ children: safeJsonArray(row.children),
124
+ summary: row.summary ?? "",
125
+ embedding: decodeEmbedding(row.embedding_blob),
126
+ qualityMarker: row.quality_marker ?? "low",
127
+ tokenEstimate: row.token_estimate ?? 0,
128
+ builtAt: Number(row.built_at ?? 0),
129
+ }));
101
130
  }
102
131
 
103
132
  /** Delete all RAPTOR nodes for a session (rollback/cleanup). */
104
- export function clearRaptorNodes(sessionId: string, stateDir: string = getStateDir()): void {
105
- const db = openStore(stateDir);
106
- db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
133
+ export function clearRaptorNodes(
134
+ sessionId: string,
135
+ stateDir: string = getStateDir(),
136
+ ): void {
137
+ const db = openStore(stateDir);
138
+ db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(
139
+ normalizeSessionId(sessionId),
140
+ );
141
+ }
142
+
143
+ /**
144
+ * S25: newest built_at for a session's RAPTOR tree (0 if no nodes exist).
145
+ * Cheap indexed MAX query used by raptorSearchHits to validate cache freshness
146
+ * without rehydrating the full node Map.
147
+ */
148
+ export function maxRaptorNodeBuiltAt(
149
+ sessionId: string,
150
+ stateDir: string = getStateDir(),
151
+ ): number {
152
+ const db = openStore(stateDir);
153
+ const row = db
154
+ .prepare(
155
+ `SELECT MAX(built_at) AS max_built FROM raptor_nodes WHERE session_id = ?`,
156
+ )
157
+ .get(normalizeSessionId(sessionId)) as
158
+ | { max_built: number | null }
159
+ | undefined;
160
+ return row?.max_built ?? 0;
107
161
  }
@@ -48,6 +48,11 @@ export function initSchema(db: DatabaseSync): void {
48
48
  CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_pk
49
49
  ON context_chunks(session_id, id);
50
50
  CREATE INDEX IF NOT EXISTS idx_chunks_session ON context_chunks(session_id);
51
+ -- S42D/QA perf: session-scoped timestamp ordering so the S25 RAPTOR
52
+ -- freshness guard's MAX(timestamp) (run on every RAPTOR search) is
53
+ -- index-satisfied rather than a full partition scan as checkpoints grow.
54
+ CREATE INDEX IF NOT EXISTS idx_chunks_session_ts
55
+ ON context_chunks(session_id, timestamp DESC);
51
56
  CREATE INDEX IF NOT EXISTS idx_chunks_region ON context_chunks(region_hash);
52
57
  CREATE INDEX IF NOT EXISTS idx_chunks_content ON context_chunks(content_hash);
53
58
  -- Partial UNIQUE (QA #1): null content_hash rows never violate the constraint;
@@ -77,7 +82,9 @@ export function initSchema(db: DatabaseSync): void {
77
82
  CREATE TABLE IF NOT EXISTS session_state (
78
83
  session_id TEXT PRIMARY KEY,
79
84
  injected_checkpoint_ids TEXT, -- JSON array
80
- stored_region_hashes TEXT -- JSON array
85
+ stored_region_hashes TEXT, -- JSON array
86
+ conversation_id TEXT, -- S43: groups turns across resumes (/clear → new root)
87
+ last_turn_id INTEGER -- S43: most recent turn id (stable fork refs)
81
88
  );
82
89
 
83
90
  CREATE TABLE IF NOT EXISTS meta (
@@ -104,6 +111,28 @@ export function initSchema(db: DatabaseSync): void {
104
111
  );
105
112
  CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
106
113
  CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
114
+ -- S42D/QA perf: session-scoped built_at ordering so the per-search cache
115
+ -- invalidation MAX(built_at) is index-satisfied.
116
+ CREATE INDEX IF NOT EXISTS idx_raptor_session_built
117
+ ON raptor_nodes(session_id, built_at DESC);
118
+
119
+ -- S42D: structured RAPTOR build history. One row per tree build; the
120
+ -- freshness check (buildHistory.ts) uses completed_at + leaf_count to skip
121
+ -- unnecessary rebuilds when the tree is recent and the chunk count is stable.
122
+ CREATE TABLE IF NOT EXISTS raptor_build_history (
123
+ build_id TEXT PRIMARY KEY,
124
+ session_id TEXT NOT NULL,
125
+ state_dir TEXT NOT NULL,
126
+ started_at INTEGER NOT NULL,
127
+ completed_at INTEGER NOT NULL,
128
+ node_count INTEGER NOT NULL,
129
+ leaf_count INTEGER NOT NULL,
130
+ depth INTEGER NOT NULL,
131
+ config_json TEXT NOT NULL, -- serialized BuildOptions
132
+ coherence_score REAL, -- avg intra-cluster cosine (post-build)
133
+ timed_out INTEGER NOT NULL DEFAULT 0
134
+ );
135
+ CREATE INDEX IF NOT EXISTS idx_raptor_build_session ON raptor_build_history(session_id);
107
136
 
108
137
  -- Foundation for future features (resume sessions, daily log, lessons
109
138
  -- learned). Scaffolded now so all store data lives in SQLite from day one;
@@ -222,6 +251,59 @@ export function initSchema(db: DatabaseSync): void {
222
251
  );
223
252
  CREATE INDEX IF NOT EXISTS idx_epoch_session ON checkpoint_epochs(session_id, created_at DESC);
224
253
 
254
+ -- S43 (per-turn vector tracking): the relational spine for per-turn +
255
+ -- per-conversation memories. One row per turn_end; links turns to the
256
+ -- epoch that compacted them (epoch_id) and to the checkpoints/cluster
257
+ -- summaries that were RECALLED during that turn (turn_recall).
258
+ -- conversation_id groups turns across pi session resumes (/clear starts a
259
+ -- new conversation root; a fork carries parent_conversation_id).
260
+ CREATE TABLE IF NOT EXISTS turns (
261
+ id INTEGER PRIMARY KEY AUTOINCREMENT, -- global turn id
262
+ conversation_id TEXT NOT NULL,
263
+ session_id TEXT NOT NULL,
264
+ turn_index INTEGER NOT NULL, -- per-session turn (event.turnIndex)
265
+ role TEXT, -- 'user' | 'assistant' | 'tool' (turn's last role)
266
+ started_at INTEGER NOT NULL,
267
+ ended_at INTEGER, -- set at turn_end
268
+ ctx_tokens INTEGER, -- runtime.lastCtxTokens snapshot
269
+ ctx_percent REAL, -- runtime.lastCtxPercent
270
+ pressure_band TEXT, -- 'low'|'mid'|'high'|'critical'
271
+ model_id TEXT,
272
+ epoch_id TEXT, -- FK checkpoint_epochs (set when a compact closes this turn's epoch)
273
+ UNIQUE(session_id, turn_index)
274
+ );
275
+ CREATE INDEX IF NOT EXISTS idx_turns_conv ON turns(conversation_id, turn_index);
276
+ CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id, turn_index);
277
+ CREATE INDEX IF NOT EXISTS idx_turns_epoch ON turns(epoch_id) WHERE epoch_id IS NOT NULL;
278
+
279
+ -- S43: recall provenance — which checkpoints/cluster summaries were
280
+ -- injected at which turn, their score, and the path that sourced them.
281
+ -- This is the per-turn vector data that makes memory quality measurable
282
+ -- per turn and enables recall-to-point (replay these checkpoint_ids into a
283
+ -- forked session). source: 'flat' | 'raptor' | 'cross-repo' | 'memory'.
284
+ CREATE TABLE IF NOT EXISTS turn_recall (
285
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
286
+ turn_id INTEGER NOT NULL,
287
+ checkpoint_id TEXT NOT NULL,
288
+ score REAL NOT NULL,
289
+ source TEXT NOT NULL,
290
+ raptor_level INTEGER, -- set for RAPTOR cluster hits
291
+ UNIQUE(turn_id, checkpoint_id)
292
+ );
293
+ CREATE INDEX IF NOT EXISTS idx_turn_recall_turn ON turn_recall(turn_id);
294
+ CREATE INDEX IF NOT EXISTS idx_turn_recall_cp ON turn_recall(checkpoint_id);
295
+
296
+ -- S43: conversation branch/fork registry. A row per fork: the child
297
+ -- conversation inherits the parent's recall state at fork_turn_id as its
298
+ -- starting injected-set. The root conversation has no row here.
299
+ CREATE TABLE IF NOT EXISTS conversation_branches (
300
+ conversation_id TEXT PRIMARY KEY,
301
+ parent_conversation_id TEXT NOT NULL,
302
+ fork_turn_id INTEGER NOT NULL, -- FK turns.id at the branch point
303
+ created_at INTEGER NOT NULL
304
+ );
305
+ CREATE INDEX IF NOT EXISTS idx_conv_branch_parent ON conversation_branches(parent_conversation_id);
306
+
225
307
  -- S27 Task 6: dedup_mirror for space-efficient deduplicated storage.
226
308
  -- Each unique content_hash stores its bytes ONCE; raw_transcript rows
227
309
  -- reference this table via content_ref instead of storing duplicate content_bytes inline.
@@ -299,6 +381,13 @@ export function initSchema(db: DatabaseSync): void {
299
381
  // S25: RAPTOR freshness-guard timestamp. Additive; old DBs have NULL → 0 →
300
382
  // treated as stale → flat fallback (safe).
301
383
  ensureColumn(db, "raptor_nodes", "built_at", "INTEGER");
384
+ // S43: turn_index on raw_transcript so a message points directly at its
385
+ // conversation turn (otherwise it must be inferred from seq ordering). NULL
386
+ // for legacy rows — turns written before S43 have no turn link.
387
+ ensureColumn(db, "raw_transcript", "turn_index", "INTEGER");
388
+ // S43: conversation_id + last_turn_id on session_state (legacy DBs have NULL).
389
+ ensureColumn(db, "session_state", "conversation_id", "TEXT");
390
+ ensureColumn(db, "session_state", "last_turn_id", "INTEGER");
302
391
  // S35: idempotent seed of the 9 achievement rows. ON CONFLICT(id) DO
303
392
  // NOTHING so a re-open never clobbers an already-unlocked row's
304
393
  // unlocked_at. No user input reaches this SQL (PREVENT-002 safe).
@@ -14,6 +14,8 @@ function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
14
14
  return {
15
15
  injectedCheckpointIds: row.injected_checkpoint_ids ? JSON.parse(row.injected_checkpoint_ids) : [],
16
16
  storedRegionHashes: row.stored_region_hashes ? JSON.parse(row.stored_region_hashes) : [],
17
+ conversationId: (row.conversation_id as string | null) ?? undefined,
18
+ lastTurnId: (row.last_turn_id as number | null) ?? undefined,
17
19
  };
18
20
  }
19
21
 
@@ -25,14 +27,18 @@ export function saveSessionState(sessionId: string, state: SessionState, stateDi
25
27
  const db = openStore(stateDir);
26
28
  const sid = normalizeSessionId(sessionId);
27
29
  db.prepare(
28
- `INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes)
29
- VALUES(@sid, @inj, @reg)
30
+ `INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes, conversation_id, last_turn_id)
31
+ VALUES(@sid, @inj, @reg, @conv, @tid)
30
32
  ON CONFLICT(session_id) DO UPDATE SET
31
33
  injected_checkpoint_ids=excluded.injected_checkpoint_ids,
32
- stored_region_hashes=excluded.stored_region_hashes`,
34
+ stored_region_hashes=excluded.stored_region_hashes,
35
+ conversation_id=excluded.conversation_id,
36
+ last_turn_id=excluded.last_turn_id`,
33
37
  ).run({
34
38
  sid,
35
39
  inj: jsonText(state.injectedCheckpointIds),
36
40
  reg: jsonText(state.storedRegionHashes),
41
+ conv: state.conversationId ?? null,
42
+ tid: state.lastTurnId ?? null,
37
43
  });
38
44
  }
@@ -18,21 +18,23 @@ export function storeStats(sessionId: string, stateDir: string = getStateDir()):
18
18
  const row = db
19
19
  .prepare(
20
20
  `SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
21
- MAX(id) AS lastId
22
- FROM context_chunks WHERE session_id = ?`,
21
+ MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS lastNum
22
+ FROM context_chunks WHERE session_id = ? AND dedup_status != 'removed'`,
23
23
  )
24
- .get(sid) as { c: number; tok: number; lastId: string | null };
24
+ .get(sid) as { c: number; tok: number; lastNum: number | null };
25
+ let lastCheckpointId: string | undefined;
25
26
  let lastSummary: string | undefined;
26
- if (row.lastId) {
27
- const s = db.prepare("SELECT summary FROM context_chunks WHERE id = ?").get(row.lastId) as
28
- | { summary: string }
29
- | undefined;
27
+ if (row.lastNum != null) {
28
+ lastCheckpointId = `chkpt_${String(row.lastNum).padStart(3, "0")}`;
29
+ const s = db
30
+ .prepare("SELECT summary FROM context_chunks WHERE session_id = ? AND id = ?")
31
+ .get(sid, lastCheckpointId) as { summary: string } | undefined;
30
32
  lastSummary = s?.summary;
31
33
  }
32
34
  return {
33
35
  checkpointCount: row.c,
34
36
  totalTokenEstimate: row.tok,
35
- lastCheckpointId: row.lastId ?? undefined,
37
+ lastCheckpointId,
36
38
  lastSummary,
37
39
  };
38
40
  }
@@ -0,0 +1,218 @@
1
+ /**
2
+ * turns.test.ts — S43 per-turn + conversation tracking tests.
3
+ *
4
+ * No network. Real stores with temp state dirs.
5
+ */
6
+
7
+ import { test, beforeEach, afterEach } from "node:test";
8
+ import assert from "node:assert/strict";
9
+ import { mkdtempSync, rmSync } from "node:fs";
10
+ import { tmpdir } from "node:os";
11
+ import { join } from "node:path";
12
+ import { openStore } from "./utils.js";
13
+ import {
14
+ recordTurn,
15
+ recordTurnRecall,
16
+ getTurn,
17
+ getTurnById,
18
+ listTurnRecall,
19
+ listConversationTurns,
20
+ ensureConversationId,
21
+ forkConversation,
22
+ newConversationId,
23
+ clearTurns,
24
+ } from "./turns.js";
25
+ import { loadSessionState, saveSessionState } from "./session-state.js";
26
+
27
+ let tmpDir: string;
28
+ let counter = 0;
29
+
30
+ beforeEach(() => {
31
+ tmpDir = mkdtempSync(join(tmpdir(), "mc-turns-"));
32
+ });
33
+
34
+ afterEach(() => {
35
+ rmSync(tmpDir, { recursive: true, force: true });
36
+ });
37
+
38
+ function stateDir(): string {
39
+ return join(tmpDir, `run-${counter++}`);
40
+ }
41
+
42
+ // ─── 1. recordTurn upserts + getTurn round-trip ────────────────────────────
43
+
44
+ test("S43-1: recordTurn inserts/updates + getTurn finds by conv+index", () => {
45
+ const sd = stateDir();
46
+ openStore(sd);
47
+ const conv = newConversationId();
48
+ const id = recordTurn({
49
+ conversationId: conv,
50
+ sessionId: "sess-1",
51
+ turnIndex: 3,
52
+ endedAt: 1000,
53
+ ctxTokens: 12000,
54
+ ctxPercent: 42.5,
55
+ pressureBand: "mid",
56
+ modelId: "claude-fable-5",
57
+ }, sd);
58
+ assert.ok(id > 0, "turn id returned");
59
+ const t = getTurn(conv, 3, sd);
60
+ assert.ok(t, "turn found");
61
+ assert.equal(t!.conversationId, conv);
62
+ assert.equal(t!.turnIndex, 3);
63
+ assert.equal(t!.ctxTokens, 12000);
64
+ assert.equal(t!.ctxPercent, 42.5);
65
+ assert.equal(t!.pressureBand, "mid");
66
+ assert.equal(t!.modelId, "claude-fable-5");
67
+ assert.equal(t!.endedAt, 1000);
68
+ // Re-record (upsert) with additional metrics — COALESCE preserves started_at.
69
+ const id2 = recordTurn({
70
+ conversationId: conv,
71
+ sessionId: "sess-1",
72
+ turnIndex: 3,
73
+ startedAt: 900,
74
+ ctxTokens: 12500,
75
+ }, sd);
76
+ assert.equal(id2, id, "same turn id on upsert");
77
+ const t2 = getTurn(conv, 3, sd);
78
+ assert.equal(t2!.ctxTokens, 12500, "metrics updated");
79
+ assert.equal(t2!.startedAt, 900, "started_at set on second write");
80
+ });
81
+
82
+ // ─── 2. recordTurnRecall + listTurnRecall ───────────────────────────────────
83
+
84
+ test("S43-2: recordTurnRecall stores provenance + listTurnRecall returns it", () => {
85
+ const sd = stateDir();
86
+ openStore(sd);
87
+ const conv = newConversationId();
88
+ const turnId = recordTurn({
89
+ conversationId: conv,
90
+ sessionId: "sess-1",
91
+ turnIndex: 1,
92
+ }, sd);
93
+ recordTurnRecall(turnId, [
94
+ { checkpointId: "chkpt_003", score: 0.91, source: "flat" },
95
+ { checkpointId: "r1_0", score: 0.78, source: "raptor", raptorLevel: 1 },
96
+ { checkpointId: "chkpt_007", score: 0.65, source: "cross-repo" },
97
+ ], sd);
98
+ const recalls = listTurnRecall(turnId, sd);
99
+ assert.equal(recalls.length, 3);
100
+ assert.equal(recalls[0].checkpointId, "chkpt_003", "sorted by score desc");
101
+ assert.equal(recalls[0].score, 0.91);
102
+ const raptorHit = recalls.find((r) => r.source === "raptor");
103
+ assert.ok(raptorHit, "raptor hit present");
104
+ assert.equal(raptorHit!.raptorLevel, 1);
105
+ assert.equal(raptorHit!.checkpointId, "r1_0");
106
+ // Re-record with a changed score → upsert, no duplicate.
107
+ recordTurnRecall(turnId, [{ checkpointId: "chkpt_003", score: 0.95, source: "flat" }], sd);
108
+ const recalls2 = listTurnRecall(turnId, sd);
109
+ assert.equal(recalls2.length, 3, "no duplicate on re-record");
110
+ const updated = recalls2.find((r) => r.checkpointId === "chkpt_003");
111
+ assert.equal(updated!.score, 0.95, "score updated");
112
+ });
113
+
114
+ // ─── 3. listConversationTurns ───────────────────────────────────────────────
115
+
116
+ test("S43-3: listConversationTurns returns turns in order", () => {
117
+ const sd = stateDir();
118
+ openStore(sd);
119
+ const conv = newConversationId();
120
+ for (let i = 1; i <= 4; i++) {
121
+ recordTurn({
122
+ conversationId: conv,
123
+ sessionId: "sess-1",
124
+ turnIndex: i,
125
+ endedAt: i * 100,
126
+ }, sd);
127
+ }
128
+ const turns = listConversationTurns(conv, sd);
129
+ assert.equal(turns.length, 4);
130
+ assert.deepEqual(turns.map((t) => t.turnIndex), [1, 2, 3, 4]);
131
+ });
132
+
133
+ // ─── 4. ensureConversationId persists + is stable across resumes ───────────
134
+
135
+ test("S43-4: ensureConversationId generates once, persists, survives reload", () => {
136
+ const sd = stateDir();
137
+ openStore(sd);
138
+ const sid = "sess-resume";
139
+ // First call generates + persists.
140
+ const conv1 = ensureConversationId(sid, sd);
141
+ assert.ok(conv1.startsWith("conv_"), "generated id has prefix");
142
+ // Second call (simulated resume) returns the same id — reads session_state.
143
+ const conv2 = ensureConversationId(sid, sd);
144
+ assert.equal(conv2, conv1, "stable across resumes");
145
+ // A different session gets a different conversation id.
146
+ const conv3 = ensureConversationId("sess-other", sd);
147
+ assert.notEqual(conv3, conv1);
148
+ });
149
+
150
+ // ─── 5. forkConversation records lineage + returns recall set ──────────────
151
+
152
+ test("S43-5: forkConversation creates child + returns parent's recall set", () => {
153
+ const sd = stateDir();
154
+ openStore(sd);
155
+ const parent = newConversationId();
156
+ // Parent conversation: turn 1 with 2 recalled checkpoints.
157
+ const turnId = recordTurn({
158
+ conversationId: parent,
159
+ sessionId: "sess-parent",
160
+ turnIndex: 1,
161
+ }, sd);
162
+ recordTurnRecall(turnId, [
163
+ { checkpointId: "chkpt_001", score: 0.9, source: "flat" },
164
+ { checkpointId: "chkpt_002", score: 0.7, source: "flat" },
165
+ ], sd);
166
+ // Fork at turn 1.
167
+ const { conversationId: child, recalled } = forkConversation(parent, turnId, sd);
168
+ assert.ok(child.startsWith("conv_"), "child is a conversation id");
169
+ assert.notEqual(child, parent, "child differs from parent");
170
+ assert.equal(recalled.length, 2, "parent's recall set returned for replay");
171
+ assert.ok(
172
+ recalled.some((r) => r.checkpointId === "chkpt_001"),
173
+ "includes parent's recalled checkpoint",
174
+ );
175
+ // The child conversation is recorded as a branch.
176
+ const branches = openStore(sd).prepare(
177
+ "SELECT parent_conversation_id, fork_turn_id FROM conversation_branches WHERE conversation_id = ?",
178
+ ).get(child) as { parent_conversation_id: string; fork_turn_id: number };
179
+ assert.equal(branches.parent_conversation_id, parent);
180
+ assert.equal(branches.fork_turn_id, turnId);
181
+ });
182
+
183
+ // ─── 6. clearTurns removes turns + their recall rows for a session ──────────
184
+
185
+ test("S43-6: clearTurns removes turns + cascade turn_recall", () => {
186
+ const sd = stateDir();
187
+ openStore(sd);
188
+ const conv = newConversationId();
189
+ const turnId = recordTurn({
190
+ conversationId: conv,
191
+ sessionId: "sess-clear",
192
+ turnIndex: 1,
193
+ }, sd);
194
+ recordTurnRecall(turnId, [{ checkpointId: "x", score: 0.5, source: "flat" }], sd);
195
+ assert.equal(listTurnRecall(turnId, sd).length, 1);
196
+ clearTurns("sess-clear", sd);
197
+ assert.equal(getTurnById(turnId, sd), null, "turn gone");
198
+ assert.equal(listTurnRecall(turnId, sd).length, 0, "recall rows cascaded");
199
+ });
200
+
201
+ // ─── 7. conversation id survives across SessionState save/load ──────────────
202
+
203
+ test("S43-7: conversationId round-trips through SessionState", () => {
204
+ const sd = stateDir();
205
+ openStore(sd);
206
+ const sid = "sess-rt";
207
+ // Save a state with a conversationId.
208
+ const conv = "conv_roundtrip";
209
+ saveSessionState(sid, {
210
+ injectedCheckpointIds: [],
211
+ storedRegionHashes: [],
212
+ conversationId: conv,
213
+ }, sd);
214
+ const loaded = loadSessionState(sid, sd);
215
+ assert.equal(loaded.conversationId, conv, "conversationId persisted");
216
+ // ensureConversationId picks up the existing one.
217
+ assert.equal(ensureConversationId(sid, sd), conv, "ensure keeps existing");
218
+ });