pi-mega-compact 0.11.12 → 0.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -63,7 +63,7 @@ function renderByModel(repos) {
63
63
  var rows = document.getElementById('bm-rows');
64
64
  if (!rows) return;
65
65
  if (!repos || !repos.length) {
66
- rows.innerHTML = '<tr><td colspan="14" class="repo-none">No repositories registered yet.</td></tr>';
66
+ rows.innerHTML = '<tr><td colspan="16" class="repo-none">No repositories registered yet.</td></tr>';
67
67
  return;
68
68
  }
69
69
  var groups = {};
@@ -106,11 +106,35 @@ function renderByModel(repos) {
106
106
  var fmt = function(v) { return '$' + v.toFixed(6); };
107
107
  return lo === hi ? fmt(lo) : fmt(lo) + '–' + fmt(hi);
108
108
  }
109
+ // Helper: compute cache hit % from providerCacheRead/Write tokens
110
+ function cacheHitPct(cr, cw, inp) {
111
+ var denom = cr + cw + (inp || 0);
112
+ return denom > 0 ? ((cr / denom) * 100).toFixed(1) + '%' : '—';
113
+ }
114
+ // Helper: compute cache $ saved (read = 0.9 * inputRate, write = 0.25 * inputRate)
115
+ function cacheDollar(cr, cw, rate) {
116
+ if (!cr || !rate) return '—';
117
+ var saved = cr * rate * 0.9 - cw * rate * 0.25;
118
+ return '$' + saved.toFixed(4);
119
+ }
120
+ // Aggregate providerCacheRead/Write across repos in this group
121
+ var groupCacheRead = 0, groupCacheWrite = 0, groupTokensIn = 0;
122
+ for (var ii = 0; ii < repos.length; ii++) {
123
+ var rr = repos[ii];
124
+ var key2 = (rr.modelName && String(rr.modelName).trim()) || '(unknown)';
125
+ if (key2 === g.model) {
126
+ groupCacheRead += (rr.providerCacheRead || 0);
127
+ groupCacheWrite += (rr.providerCacheWrite || 0);
128
+ groupTokensIn += (rr.tokensDropped || 0);
129
+ }
130
+ }
109
131
  rows.innerHTML = arr.map(function(g) {
110
132
  var freed = (g.tokensIn || 0) - (g.tokensOut || 0);
111
- var usd = g.usd > 0 ? '$' + g.usd.toFixed(4) : '—';
133
+ // B: always show a dollar value (including $0.0000) do not use '' when usd is 0
134
+ var usd = '$' + (g.usd || 0).toFixed(4);
112
135
  var when = g.lastAt ? new Date(g.lastAt).toLocaleString() : '—';
113
136
  var reas = g.reasoning == null ? '—' : (g.reasoning ? 'yes' : 'no');
137
+ var rate0 = g.inRates && g.inRates.length ? g.inRates[0] : 0;
114
138
  return '<tr>' +
115
139
  '<td><span class="repo-model">' + sanitize(g.model) + '</span></td>' +
116
140
  '<td>' + sanitize(g.provider) + '</td>' +
@@ -125,6 +149,8 @@ function renderByModel(repos) {
125
149
  '<td class="num">' + collapseRate(g.inRates) + '</td>' +
126
150
  '<td class="num">' + collapseRate(g.outRates) + '</td>' +
127
151
  '<td class="num">' + sanitize(usd) + '</td>' +
152
+ '<td class="num">' + cacheHitPct(groupCacheRead, groupCacheWrite, groupTokensIn) + '</td>' +
153
+ '<td class="num">' + cacheDollar(groupCacheRead, groupCacheWrite, rate0) + '</td>' +
128
154
  '<td class="num">' + sanitize(when) + '</td>' +
129
155
  '</tr>';
130
156
  }).join('');
@@ -12,6 +12,7 @@ import { existsSync } from "node:fs";
12
12
  import { homedir } from "node:os";
13
13
  import { join } from "node:path";
14
14
  import { DatabaseSync } from "node:sqlite";
15
+ import { readProviderCacheForRepo, } from "../../src/store/sqlite/perf-samples.js";
15
16
  export function getIndexDir() {
16
17
  const override = process.env.MEGACOMPACT_INDEX_DIR;
17
18
  if (override && override.trim() !== "")
@@ -89,21 +90,21 @@ export function readIndex() {
89
90
  repo.maxTokens = Number(mrow.max_tokens ?? 0) || null;
90
91
  repo.reasoning = Number(mrow.reasoning ?? 0) === 1;
91
92
  }
92
- // Provider prompt-cache stats (E.3)
93
+ // Provider prompt-cache stats (E.3 / F4)
94
+ // Use readProviderCacheForRepo which correctly reads cache token
95
+ // counts from the meta JSON column (cacheRead/cacheWrite), not
96
+ // from non-existent table columns cache_read/cache_write.
93
97
  try {
94
- const pRow = sdb
95
- .prepare(`SELECT
96
- SUM(cache_read) AS totalRead,
97
- SUM(cache_write) AS totalWrite,
98
- SUM(cache_read + cache_write) * 1.0 / MAX(SUM(cache_read + cache_write + tokens_in), 1) AS hitPct
99
- FROM perf_samples
100
- WHERE cache_read > 0 OR cache_write > 0`)
101
- .get();
102
- if (pRow && pRow.totalRead > 0) {
103
- repo.providerCacheRead = Number(pRow.totalRead);
104
- repo.providerCacheWrite = Number(pRow.totalWrite);
105
- repo.providerCachePct = Number((pRow.hitPct * 100).toFixed(1));
98
+ const pc = readProviderCacheForRepo(repo.stateDir);
99
+ if (pc.totalCacheRead > 0) {
100
+ repo.providerCacheRead = pc.totalCacheRead;
101
+ repo.providerCacheWrite = pc.totalCacheWrite;
102
+ repo.providerCachePct = Number(pc.avgHitPct.toFixed(1));
106
103
  }
104
+ // Also enrich contextWindow/maxTokens/reasoning from the
105
+ // latest model_snapshots row (was already read above, but
106
+ // when latestModelSnapshot was used above we get the same
107
+ // data from the per-repo store — keep as-is).
107
108
  }
108
109
  catch {
109
110
  /* best-effort — perf_samples may not exist */
@@ -7,6 +7,8 @@
7
7
  */
8
8
  import { openTurnStore } from "../../src/store/turns/connection.js";
9
9
  import { createTopicStore } from "../../src/topics/store.js";
10
+ import { openStore } from "../../src/store/sqlite.js";
11
+ import { buildTopicModel } from "../../src/topics/cluster.js";
10
12
  export function handleTopics(req, res, ctx) {
11
13
  const url = req.url ?? "";
12
14
  // ── GET /api/topics/:topicId/memories — wiki topic drill-down (S52) ──
@@ -42,6 +44,22 @@ export function handleTopics(req, res, ctx) {
42
44
  return false;
43
45
  try {
44
46
  const tdb = openTurnStore(ctx.stateDir);
47
+ // Lazy build: when the topics table is empty, attempt a one-shot build
48
+ // from available context_chunks embeddings so the wiki is useful even
49
+ // before the first compaction fires.
50
+ const topicCount = tdb.prepare("SELECT COUNT(*) AS c FROM topics").get().c;
51
+ if (topicCount === 0) {
52
+ try {
53
+ const mainDb = openStore(ctx.stateDir);
54
+ const model = buildTopicModel(mainDb);
55
+ if (model.k > 0 && model.totalChunks > 0) {
56
+ createTopicStore(ctx.stateDir).replaceTopicModel(model);
57
+ }
58
+ }
59
+ catch {
60
+ /* non-fatal: lazy wiki build is best-effort */
61
+ }
62
+ }
45
63
  const topics = tdb
46
64
  .prepare(`SELECT id, label, term_scores, memory_count, last_updated
47
65
  FROM topics ORDER BY memory_count DESC, id ASC`)
@@ -278,6 +278,11 @@ export function registerContextHandler(pi, runtime, config) {
278
278
  // S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
279
279
  // from real context_chunks embeddings. Isolated-store only, gated on
280
280
  // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
281
+ // Fire regardless of dbMirror — context_chunks is the isolated store.
282
+ // Uses the already-open db when inside the dbMirror block; opens its own
283
+ // connection otherwise.
284
+ // from real context_chunks embeddings. Isolated-store only, gated on
285
+ // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
281
286
  try {
282
287
  if (config.autoWikiEnabled && config.turnsDbEnabled) {
283
288
  const every = Math.max(1, TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS);
@@ -323,11 +328,11 @@ export function registerContextHandler(pi, runtime, config) {
323
328
  if (chunkCount < floor) {
324
329
  const transcriptRows = db
325
330
  .prepare(`SELECT DISTINCT content_bytes
326
- FROM raw_transcript
327
- WHERE session_id = ?
328
- AND length(content_bytes) > 10
329
- ORDER BY seq ASC
330
- LIMIT 200`)
331
+ FROM raw_transcript
332
+ WHERE session_id = ?
333
+ AND length(content_bytes) > 10
334
+ ORDER BY seq ASC
335
+ LIMIT 200`)
331
336
  .all(runtime.rt.sessionId);
332
337
  if (transcriptRows.length > 0) {
333
338
  const embedder = new TrigramEmbedder();
@@ -11,7 +11,11 @@
11
11
  import { normalizeSessionId } from "../../src/store.js";
12
12
  // --------------------------------------------------------------- resetRuntime
13
13
  export function resetRuntimeImpl(self, sessionId) {
14
- const sid = normalizeSessionId(sessionId);
14
+ // Only call normalizeSessionId when a real sessionId string is passed.
15
+ // When undefined, keep the existing self.rt.sessionId (set by the real
16
+ // session-start / compact pipeline) so the early-return guard can fire and
17
+ // dashboard stats keyed by sessionId are not silently orphaned.
18
+ const sid = sessionId ? normalizeSessionId(sessionId) : self.rt.sessionId;
15
19
  if (self.rt.sessionId === sid && self.rt.persistedThisSession)
16
20
  return; // same session, keep checkpoint memory
17
21
  self.rt = {
@@ -89,7 +89,7 @@ export function gatePromotionGuard(ws) {
89
89
  continue;
90
90
  }
91
91
  const epochId = n.epochId;
92
- if (epochId && checkpointIds.has(epochId)) {
92
+ if (epochId && checkpointIds.size > 0 && checkpointIds.has(epochId)) {
93
93
  droppedIds.add(n.id);
94
94
  dropped++;
95
95
  log.warn("graph_orphaned_epoch", {
@@ -27,14 +27,22 @@ export function areMemoriesEnabled() {
27
27
  // Source: checkpoints (from context_chunks table)
28
28
  // ---------------------------------------------------------------------------
29
29
  export function buildCheckpointNodes(db, sessionId, nodes, edges) {
30
- const rows = db
31
- .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
32
- dedup_status, topic_summary, key_decisions,
33
- normalized_text, embedding_blob
34
- FROM context_chunks
35
- WHERE session_id = ?
36
- ORDER BY timestamp ASC`)
37
- .all(sessionId);
30
+ const rows = sessionId
31
+ ? db
32
+ .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
33
+ dedup_status, topic_summary, key_decisions,
34
+ normalized_text, embedding_blob
35
+ FROM context_chunks
36
+ WHERE session_id = ?
37
+ ORDER BY timestamp ASC`)
38
+ .all(sessionId)
39
+ : db
40
+ .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
41
+ dedup_status, topic_summary, key_decisions,
42
+ normalized_text, embedding_blob
43
+ FROM context_chunks
44
+ ORDER BY timestamp ASC`)
45
+ .all();
38
46
  if (rows.length === 0)
39
47
  return;
40
48
  let prevId = null;
@@ -97,13 +105,20 @@ function addCheckpointSemanticEdges(rows, edges) {
97
105
  // ---------------------------------------------------------------------------
98
106
  export function buildTurnNodes(db, sessionId, nodes, edges) {
99
107
  const existingIds = new Set(nodes.map((n) => n.id));
100
- const rows = db
101
- .prepare(`SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
102
- epoch_id, ended_at
103
- FROM turns
104
- WHERE session_id = ?
105
- ORDER BY turn_index ASC`)
106
- .all(sessionId);
108
+ const rows = sessionId
109
+ ? db
110
+ .prepare(`SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
111
+ epoch_id, ended_at
112
+ FROM turns
113
+ WHERE session_id = ?
114
+ ORDER BY turn_index ASC`)
115
+ .all(sessionId)
116
+ : db
117
+ .prepare(`SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
118
+ epoch_id, ended_at
119
+ FROM turns
120
+ ORDER BY turn_index ASC`)
121
+ .all();
107
122
  if (rows.length === 0)
108
123
  return;
109
124
  let prevId = null;
@@ -146,13 +161,20 @@ export function buildTurnContentNodes(db, sessionId, nodes, edges) {
146
161
  // (identity_merge) merges them with the richest nodeType winning. Using the
147
162
  // global nodes array here would skip every node (Source A already added them).
148
163
  const seen = new Set();
149
- const rows = db
150
- .prepare(`SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
151
- FROM turns t
152
- JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
153
- WHERE t.session_id = ?
154
- ORDER BY t.turn_index ASC`)
155
- .all(sessionId);
164
+ const rows = sessionId
165
+ ? db
166
+ .prepare(`SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
167
+ FROM turns t
168
+ JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
169
+ WHERE t.session_id = ?
170
+ ORDER BY t.turn_index ASC`)
171
+ .all(sessionId)
172
+ : db
173
+ .prepare(`SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
174
+ FROM turns t
175
+ JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
176
+ ORDER BY t.turn_index ASC`)
177
+ .all();
156
178
  if (rows.length === 0)
157
179
  return;
158
180
  const turnContentNodes = [];
@@ -176,9 +176,9 @@ export class SqliteTurnStore {
176
176
  try {
177
177
  this.db
178
178
  .prepare(`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
179
- ctx_tokens, ctx_percent, pressure_band, model)
180
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
181
- .run(entry.conversationId, sid, entry.turnIndex, entry.role, entry.endedAt, entry.ctxTokens ?? null, entry.ctxPercent ?? null, entry.pressureBand ?? null, entry.model ?? null);
179
+ ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
180
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
181
+ .run(entry.conversationId, sid, entry.turnIndex, entry.role, entry.endedAt, entry.ctxTokens ?? null, entry.ctxPercent ?? null, entry.pressureBand ?? null, entry.model ?? null, entry.epochId ?? null);
182
182
  }
183
183
  catch (e) {
184
184
  if (e instanceof Error &&
@@ -364,8 +364,8 @@ export class SqliteTurnStore {
364
364
  restore(from) {
365
365
  this.clear();
366
366
  const insertTurn = this.db.prepare(`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
367
- ctx_tokens, ctx_percent, pressure_band, model)
368
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
367
+ ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
368
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
369
369
  const insertRecall = this.db.prepare(`INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
370
370
  VALUES (?, ?, ?, ?, ?)`);
371
371
  const insertFork = this.db.prepare(`INSERT INTO conversation_forks (parent_conversation_id, child_conversation_id, fork_turn_index, created_at)
@@ -376,7 +376,7 @@ export class SqliteTurnStore {
376
376
  const posToNewId = new Map();
377
377
  for (let i = 0; i < from.turns.length; i++) {
378
378
  const t = from.turns[i];
379
- insertTurn.run(t.conversationId, normalizeSessionId(t.sessionId), t.turnIndex, t.role, t.endedAt, t.ctxTokens ?? null, t.ctxPercent ?? null, t.pressureBand ?? null, t.model ?? null);
379
+ insertTurn.run(t.conversationId, normalizeSessionId(t.sessionId), t.turnIndex, t.role, t.endedAt, t.ctxTokens ?? null, t.ctxPercent ?? null, t.pressureBand ?? null, t.model ?? null, t.epochId ?? null);
380
380
  const row = this.db
381
381
  .prepare("SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?")
382
382
  .get(t.conversationId, t.turnIndex);
@@ -64,7 +64,7 @@ function renderByModel(repos) {
64
64
  var rows = document.getElementById('bm-rows');
65
65
  if (!rows) return;
66
66
  if (!repos || !repos.length) {
67
- rows.innerHTML = '<tr><td colspan="14" class="repo-none">No repositories registered yet.</td></tr>';
67
+ rows.innerHTML = '<tr><td colspan="16" class="repo-none">No repositories registered yet.</td></tr>';
68
68
  return;
69
69
  }
70
70
  var groups = {};
@@ -107,11 +107,35 @@ function renderByModel(repos) {
107
107
  var fmt = function(v) { return '$' + v.toFixed(6); };
108
108
  return lo === hi ? fmt(lo) : fmt(lo) + '–' + fmt(hi);
109
109
  }
110
+ // Helper: compute cache hit % from providerCacheRead/Write tokens
111
+ function cacheHitPct(cr, cw, inp) {
112
+ var denom = cr + cw + (inp || 0);
113
+ return denom > 0 ? ((cr / denom) * 100).toFixed(1) + '%' : '—';
114
+ }
115
+ // Helper: compute cache $ saved (read = 0.9 * inputRate, write = 0.25 * inputRate)
116
+ function cacheDollar(cr, cw, rate) {
117
+ if (!cr || !rate) return '—';
118
+ var saved = cr * rate * 0.9 - cw * rate * 0.25;
119
+ return '$' + saved.toFixed(4);
120
+ }
121
+ // Aggregate providerCacheRead/Write across repos in this group
122
+ var groupCacheRead = 0, groupCacheWrite = 0, groupTokensIn = 0;
123
+ for (var ii = 0; ii < repos.length; ii++) {
124
+ var rr = repos[ii];
125
+ var key2 = (rr.modelName && String(rr.modelName).trim()) || '(unknown)';
126
+ if (key2 === g.model) {
127
+ groupCacheRead += (rr.providerCacheRead || 0);
128
+ groupCacheWrite += (rr.providerCacheWrite || 0);
129
+ groupTokensIn += (rr.tokensDropped || 0);
130
+ }
131
+ }
110
132
  rows.innerHTML = arr.map(function(g) {
111
133
  var freed = (g.tokensIn || 0) - (g.tokensOut || 0);
112
- var usd = g.usd > 0 ? '$' + g.usd.toFixed(4) : '—';
134
+ // B: always show a dollar value (including $0.0000) do not use '' when usd is 0
135
+ var usd = '$' + (g.usd || 0).toFixed(4);
113
136
  var when = g.lastAt ? new Date(g.lastAt).toLocaleString() : '—';
114
137
  var reas = g.reasoning == null ? '—' : (g.reasoning ? 'yes' : 'no');
138
+ var rate0 = g.inRates && g.inRates.length ? g.inRates[0] : 0;
115
139
  return '<tr>' +
116
140
  '<td><span class="repo-model">' + sanitize(g.model) + '</span></td>' +
117
141
  '<td>' + sanitize(g.provider) + '</td>' +
@@ -126,6 +150,8 @@ function renderByModel(repos) {
126
150
  '<td class="num">' + collapseRate(g.inRates) + '</td>' +
127
151
  '<td class="num">' + collapseRate(g.outRates) + '</td>' +
128
152
  '<td class="num">' + sanitize(usd) + '</td>' +
153
+ '<td class="num">' + cacheHitPct(groupCacheRead, groupCacheWrite, groupTokensIn) + '</td>' +
154
+ '<td class="num">' + cacheDollar(groupCacheRead, groupCacheWrite, rate0) + '</td>' +
129
155
  '<td class="num">' + sanitize(when) + '</td>' +
130
156
  '</tr>';
131
157
  }).join('');
@@ -14,6 +14,9 @@ import { homedir } from "node:os";
14
14
  import { join } from "node:path";
15
15
  import { DatabaseSync } from "node:sqlite";
16
16
  import type { IndexRepo, IndexIndex } from "./types.js";
17
+ import {
18
+ readProviderCacheForRepo,
19
+ } from "../../src/store/sqlite/perf-samples.js";
17
20
 
18
21
  export function getIndexDir(): string {
19
22
  const override = process.env.MEGACOMPACT_INDEX_DIR;
@@ -100,25 +103,21 @@ export function readIndex(): IndexIndex | null {
100
103
  repo.maxTokens = Number(mrow.max_tokens ?? 0) || null;
101
104
  repo.reasoning = Number(mrow.reasoning ?? 0) === 1;
102
105
  }
103
- // Provider prompt-cache stats (E.3)
106
+ // Provider prompt-cache stats (E.3 / F4)
107
+ // Use readProviderCacheForRepo which correctly reads cache token
108
+ // counts from the meta JSON column (cacheRead/cacheWrite), not
109
+ // from non-existent table columns cache_read/cache_write.
104
110
  try {
105
- const pRow = sdb
106
- .prepare(
107
- `SELECT
108
- SUM(cache_read) AS totalRead,
109
- SUM(cache_write) AS totalWrite,
110
- SUM(cache_read + cache_write) * 1.0 / MAX(SUM(cache_read + cache_write + tokens_in), 1) AS hitPct
111
- FROM perf_samples
112
- WHERE cache_read > 0 OR cache_write > 0`,
113
- )
114
- .get() as
115
- | { totalRead: number; totalWrite: number; hitPct: number }
116
- | undefined;
117
- if (pRow && pRow.totalRead > 0) {
118
- repo.providerCacheRead = Number(pRow.totalRead);
119
- repo.providerCacheWrite = Number(pRow.totalWrite);
120
- repo.providerCachePct = Number((pRow.hitPct * 100).toFixed(1));
111
+ const pc = readProviderCacheForRepo(repo.stateDir);
112
+ if (pc.totalCacheRead > 0) {
113
+ repo.providerCacheRead = pc.totalCacheRead;
114
+ repo.providerCacheWrite = pc.totalCacheWrite;
115
+ repo.providerCachePct = Number(pc.avgHitPct.toFixed(1));
121
116
  }
117
+ // Also enrich contextWindow/maxTokens/reasoning from the
118
+ // latest model_snapshots row (was already read above, but
119
+ // when latestModelSnapshot was used above we get the same
120
+ // data from the per-repo store — keep as-is).
122
121
  } catch {
123
122
  /* best-effort — perf_samples may not exist */
124
123
  }
@@ -10,6 +10,8 @@ import type { IncomingMessage, ServerResponse } from "node:http";
10
10
  import type { RouteContext } from "./routes-core.js";
11
11
  import { openTurnStore } from "../../src/store/turns/connection.js";
12
12
  import { createTopicStore } from "../../src/topics/store.js";
13
+ import { openStore } from "../../src/store/sqlite.js";
14
+ import { buildTopicModel } from "../../src/topics/cluster.js";
13
15
  import type { TopicsResponse, TopicRow } from "./api-contracts/game-types.js";
14
16
  import type { TopicMemoriesResponse } from "./api-contracts/turns.js";
15
17
 
@@ -53,6 +55,25 @@ export function handleTopics(
53
55
 
54
56
  try {
55
57
  const tdb = openTurnStore(ctx.stateDir);
58
+
59
+ // Lazy build: when the topics table is empty, attempt a one-shot build
60
+ // from available context_chunks embeddings so the wiki is useful even
61
+ // before the first compaction fires.
62
+ const topicCount = (
63
+ tdb.prepare("SELECT COUNT(*) AS c FROM topics").get() as { c: number }
64
+ ).c;
65
+ if (topicCount === 0) {
66
+ try {
67
+ const mainDb = openStore(ctx.stateDir);
68
+ const model = buildTopicModel(mainDb);
69
+ if (model.k > 0 && model.totalChunks > 0) {
70
+ createTopicStore(ctx.stateDir).replaceTopicModel(model);
71
+ }
72
+ } catch {
73
+ /* non-fatal: lazy wiki build is best-effort */
74
+ }
75
+ }
76
+
56
77
  const topics: TopicRow[] = (
57
78
  tdb
58
79
  .prepare(
@@ -333,120 +333,127 @@ export function registerContextHandler(
333
333
  } catch {
334
334
  /* non-fatal: epoch stamping never breaks compaction */
335
335
  }
336
- // S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
337
- // from real context_chunks embeddings. Isolated-store only, gated on
338
- // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
339
- try {
340
- if (config.autoWikiEnabled && config.turnsDbEnabled) {
341
- const every = Math.max(
342
- 1,
343
- TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
344
- );
345
- const tdb = openTurnStore(runtime.currentStateDir);
346
- const n = bumpWikiCompactCounter(tdb);
347
- if (n % every === 0) {
348
- const model = buildTopicModel(db, {
349
- kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
350
- labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
351
- restarts: 5,
352
- seed: 0x9e3779b9,
336
+
337
+ // S51B: auto-categorizing wiki rebuild every Nth compaction, derived
338
+ // from real context_chunks embeddings. Isolated-store only, gated on
339
+ // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
340
+ // Fire regardless of dbMirror — context_chunks is the isolated store.
341
+ // Uses the already-open db when inside the dbMirror block; opens its own
342
+ // connection otherwise.
343
+ // from real context_chunks embeddings. Isolated-store only, gated on
344
+ // AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
345
+ try {
346
+ if (config.autoWikiEnabled && config.turnsDbEnabled) {
347
+ const every = Math.max(
348
+ 1,
349
+ TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
350
+ );
351
+ const tdb = openTurnStore(runtime.currentStateDir);
352
+ const n = bumpWikiCompactCounter(tdb);
353
+ if (n % every === 0) {
354
+ const model = buildTopicModel(db, {
355
+ kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
356
+ labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
357
+ restarts: 5,
358
+ seed: 0x9e3779b9,
359
+ });
360
+ createTopicStore(runtime.currentStateDir).replaceTopicModel(
361
+ model,
362
+ );
363
+ runtime.logger.info("wiki_rebuild", {
364
+ clusterCount: model.k,
365
+ totalChunks: model.totalChunks,
366
+ method: "kmeans+tfidf",
367
+ criterion: model.criterion,
368
+ silhouetteScore: model.silhouetteScore,
369
+ uncalibrated: false,
370
+ });
371
+ }
372
+ }
373
+ } catch (wikiErr) {
374
+ runtime.logger.warn("wiki_rebuild_failed", {
375
+ error: String(wikiErr),
376
+ });
377
+ }
378
+
379
+ // D1: seed the topic model from raw_transcript when context_chunks is
380
+ // thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
381
+ // Seeds buildTopicModel with on-the-fly trigram embeddings from
382
+ // recent raw_transcript rows for the current session.
383
+ try {
384
+ if (
385
+ config.autoWikiEnabled &&
386
+ config.turnsDbEnabled &&
387
+ TurnsConfig.WIKI_SEED_FROM_TURNS
388
+ ) {
389
+ const floor = 50;
390
+ const countRow = db
391
+ .prepare(
392
+ `SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
393
+ )
394
+ .get(runtime.rt.sessionId) as { cnt: number } | undefined;
395
+ const chunkCount = countRow?.cnt ?? 0;
396
+ if (chunkCount < floor) {
397
+ const transcriptRows = db
398
+ .prepare(
399
+ `SELECT DISTINCT content_bytes
400
+ FROM raw_transcript
401
+ WHERE session_id = ?
402
+ AND length(content_bytes) > 10
403
+ ORDER BY seq ASC
404
+ LIMIT 200`,
405
+ )
406
+ .all(runtime.rt.sessionId) as Array<{
407
+ content_bytes: string;
408
+ }>;
409
+ if (transcriptRows.length > 0) {
410
+ const embedder = new TrigramEmbedder();
411
+ const seedChunks: EmbeddedChunk[] = [];
412
+ for (let i = 0; i < transcriptRows.length; i++) {
413
+ const text = transcriptRows[i].content_bytes.trim();
414
+ if (text.length === 0) continue;
415
+ const vec = embedder.embed(text);
416
+ seedChunks.push({
417
+ chunkId: `seed_transcript_${i}`,
418
+ sessionId: runtime.rt.sessionId,
419
+ vec,
420
+ text,
353
421
  });
354
- createTopicStore(runtime.currentStateDir).replaceTopicModel(
355
- model,
422
+ }
423
+ if (seedChunks.length > 0) {
424
+ const model = buildTopicModel(
425
+ db,
426
+ {
427
+ kRange: [
428
+ TurnsConfig.WIKI_K_MIN,
429
+ TurnsConfig.WIKI_K_MAX,
430
+ ],
431
+ labelTopTerms:
432
+ TurnsConfig.WIKI_LABEL_TOP_TERMS,
433
+ restarts: 5,
434
+ seed: 0x9e3779b9,
435
+ },
436
+ seedChunks,
356
437
  );
357
- runtime.logger.info("wiki_rebuild", {
438
+ createTopicStore(
439
+ runtime.currentStateDir,
440
+ ).replaceTopicModel(model);
441
+ runtime.logger.info("wiki_seed", {
358
442
  clusterCount: model.k,
443
+ sourceChunks: seedChunks.length,
359
444
  totalChunks: model.totalChunks,
360
445
  method: "kmeans+tfidf",
361
- criterion: model.criterion,
362
- silhouetteScore: model.silhouetteScore,
363
- uncalibrated: false,
364
446
  });
365
447
  }
366
448
  }
367
- } catch (wikiErr) {
368
- runtime.logger.warn("wiki_rebuild_failed", {
369
- error: String(wikiErr),
370
- });
371
449
  }
450
+ }
451
+ } catch (seedErr) {
452
+ runtime.logger.warn("wiki_seed_failed", {
453
+ error: String(seedErr),
454
+ });
455
+ }
372
456
 
373
- // D1: seed the topic model from raw_transcript when context_chunks is
374
- // thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
375
- // Seeds buildTopicModel with on-the-fly trigram embeddings from
376
- // recent raw_transcript rows for the current session.
377
- try {
378
- if (
379
- config.autoWikiEnabled &&
380
- config.turnsDbEnabled &&
381
- TurnsConfig.WIKI_SEED_FROM_TURNS
382
- ) {
383
- const floor = 50;
384
- const countRow = db
385
- .prepare(
386
- `SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
387
- )
388
- .get(runtime.rt.sessionId) as { cnt: number } | undefined;
389
- const chunkCount = countRow?.cnt ?? 0;
390
- if (chunkCount < floor) {
391
- const transcriptRows = db
392
- .prepare(
393
- `SELECT DISTINCT content_bytes
394
- FROM raw_transcript
395
- WHERE session_id = ?
396
- AND length(content_bytes) > 10
397
- ORDER BY seq ASC
398
- LIMIT 200`,
399
- )
400
- .all(runtime.rt.sessionId) as Array<{
401
- content_bytes: string;
402
- }>;
403
- if (transcriptRows.length > 0) {
404
- const embedder = new TrigramEmbedder();
405
- const seedChunks: EmbeddedChunk[] = [];
406
- for (let i = 0; i < transcriptRows.length; i++) {
407
- const text = transcriptRows[i].content_bytes.trim();
408
- if (text.length === 0) continue;
409
- const vec = embedder.embed(text);
410
- seedChunks.push({
411
- chunkId: `seed_transcript_${i}`,
412
- sessionId: runtime.rt.sessionId,
413
- vec,
414
- text,
415
- });
416
- }
417
- if (seedChunks.length > 0) {
418
- const model = buildTopicModel(
419
- db,
420
- {
421
- kRange: [
422
- TurnsConfig.WIKI_K_MIN,
423
- TurnsConfig.WIKI_K_MAX,
424
- ],
425
- labelTopTerms:
426
- TurnsConfig.WIKI_LABEL_TOP_TERMS,
427
- restarts: 5,
428
- seed: 0x9e3779b9,
429
- },
430
- seedChunks,
431
- );
432
- createTopicStore(
433
- runtime.currentStateDir,
434
- ).replaceTopicModel(model);
435
- runtime.logger.info("wiki_seed", {
436
- clusterCount: model.k,
437
- sourceChunks: seedChunks.length,
438
- totalChunks: model.totalChunks,
439
- method: "kmeans+tfidf",
440
- });
441
- }
442
- }
443
- }
444
- }
445
- } catch (seedErr) {
446
- runtime.logger.warn("wiki_seed_failed", {
447
- error: String(seedErr),
448
- });
449
- }
450
457
  // S27 Task 6: Fire-and-forget dedup pipeline.
451
458
  // Deduplicates raw_transcript rows for the compacted range.
452
459
  try {
@@ -42,7 +42,11 @@ export function resetRuntimeImpl(
42
42
  self: ResetRuntimeContext,
43
43
  sessionId: string | undefined,
44
44
  ): void {
45
- const sid = normalizeSessionId(sessionId);
45
+ // Only call normalizeSessionId when a real sessionId string is passed.
46
+ // When undefined, keep the existing self.rt.sessionId (set by the real
47
+ // session-start / compact pipeline) so the early-return guard can fire and
48
+ // dashboard stats keyed by sessionId are not silently orphaned.
49
+ const sid = sessionId ? normalizeSessionId(sessionId) : self.rt.sessionId;
46
50
  if (self.rt.sessionId === sid && self.rt.persistedThisSession) return; // same session, keep checkpoint memory
47
51
  self.rt = {
48
52
  sessionId: sid,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.11.12",
3
+ "version": "0.11.13",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -112,7 +112,7 @@ export function gatePromotionGuard(ws: GraphWorkingSet): GateResult {
112
112
  continue;
113
113
  }
114
114
  const epochId = n.epochId;
115
- if (epochId && checkpointIds.has(epochId)) {
115
+ if (epochId && checkpointIds.size > 0 && checkpointIds.has(epochId)) {
116
116
  droppedIds.add(n.id);
117
117
  dropped++;
118
118
  log.warn("graph_orphaned_epoch", {
@@ -51,16 +51,26 @@ export function buildCheckpointNodes(
51
51
  nodes: MemoryGraphNode[],
52
52
  edges: MemoryGraphEdge[],
53
53
  ): void {
54
- const rows = db
55
- .prepare(
56
- `SELECT id, session_id, summary, token_estimate, timestamp,
57
- dedup_status, topic_summary, key_decisions,
58
- normalized_text, embedding_blob
59
- FROM context_chunks
60
- WHERE session_id = ?
61
- ORDER BY timestamp ASC`,
62
- )
63
- .all(sessionId) as Array<Record<string, unknown>>;
54
+ const rows = sessionId
55
+ ? (db
56
+ .prepare(
57
+ `SELECT id, session_id, summary, token_estimate, timestamp,
58
+ dedup_status, topic_summary, key_decisions,
59
+ normalized_text, embedding_blob
60
+ FROM context_chunks
61
+ WHERE session_id = ?
62
+ ORDER BY timestamp ASC`,
63
+ )
64
+ .all(sessionId) as Array<Record<string, unknown>>)
65
+ : (db
66
+ .prepare(
67
+ `SELECT id, session_id, summary, token_estimate, timestamp,
68
+ dedup_status, topic_summary, key_decisions,
69
+ normalized_text, embedding_blob
70
+ FROM context_chunks
71
+ ORDER BY timestamp ASC`,
72
+ )
73
+ .all() as Array<Record<string, unknown>>);
64
74
 
65
75
  if (rows.length === 0) return;
66
76
 
@@ -136,15 +146,24 @@ export function buildTurnNodes(
136
146
  edges: MemoryGraphEdge[],
137
147
  ): void {
138
148
  const existingIds = new Set(nodes.map((n) => n.id));
139
- const rows = db
140
- .prepare(
141
- `SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
142
- epoch_id, ended_at
143
- FROM turns
144
- WHERE session_id = ?
145
- ORDER BY turn_index ASC`,
146
- )
147
- .all(sessionId) as Array<Record<string, unknown>>;
149
+ const rows = sessionId
150
+ ? (db
151
+ .prepare(
152
+ `SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
153
+ epoch_id, ended_at
154
+ FROM turns
155
+ WHERE session_id = ?
156
+ ORDER BY turn_index ASC`,
157
+ )
158
+ .all(sessionId) as Array<Record<string, unknown>>)
159
+ : (db
160
+ .prepare(
161
+ `SELECT turn_index, role, pressure_band, ctx_tokens, ctx_percent,
162
+ epoch_id, ended_at
163
+ FROM turns
164
+ ORDER BY turn_index ASC`,
165
+ )
166
+ .all() as Array<Record<string, unknown>>);
148
167
 
149
168
  if (rows.length === 0) return;
150
169
 
@@ -197,15 +216,24 @@ export function buildTurnContentNodes(
197
216
  // (identity_merge) merges them with the richest nodeType winning. Using the
198
217
  // global nodes array here would skip every node (Source A already added them).
199
218
  const seen = new Set<string>();
200
- const rows = db
201
- .prepare(
202
- `SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
203
- FROM turns t
204
- JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
205
- WHERE t.session_id = ?
206
- ORDER BY t.turn_index ASC`,
207
- )
208
- .all(sessionId) as Array<Record<string, unknown>>;
219
+ const rows = sessionId
220
+ ? (db
221
+ .prepare(
222
+ `SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
223
+ FROM turns t
224
+ JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
225
+ WHERE t.session_id = ?
226
+ ORDER BY t.turn_index ASC`,
227
+ )
228
+ .all(sessionId) as Array<Record<string, unknown>>)
229
+ : (db
230
+ .prepare(
231
+ `SELECT t.turn_index, t.role, t.ended_at, r.content_bytes
232
+ FROM turns t
233
+ JOIN raw_transcript r ON r.session_id = t.session_id AND r.turn_index = t.turn_index
234
+ ORDER BY t.turn_index ASC`,
235
+ )
236
+ .all() as Array<Record<string, unknown>>);
209
237
 
210
238
  if (rows.length === 0) return;
211
239
 
@@ -242,8 +242,8 @@ export class SqliteTurnStore implements TurnStore {
242
242
  this.db
243
243
  .prepare(
244
244
  `INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
245
- ctx_tokens, ctx_percent, pressure_band, model)
246
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
245
+ ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
246
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
247
247
  )
248
248
  .run(
249
249
  entry.conversationId,
@@ -255,6 +255,7 @@ export class SqliteTurnStore implements TurnStore {
255
255
  entry.ctxPercent ?? null,
256
256
  entry.pressureBand ?? null,
257
257
  entry.model ?? null,
258
+ entry.epochId ?? null,
258
259
  );
259
260
  } catch (e) {
260
261
  if (
@@ -524,8 +525,8 @@ export class SqliteTurnStore implements TurnStore {
524
525
  this.clear();
525
526
  const insertTurn = this.db.prepare(
526
527
  `INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
527
- ctx_tokens, ctx_percent, pressure_band, model)
528
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
528
+ ctx_tokens, ctx_percent, pressure_band, model, epoch_id)
529
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
529
530
  );
530
531
  const insertRecall = this.db.prepare(
531
532
  `INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
@@ -553,6 +554,7 @@ export class SqliteTurnStore implements TurnStore {
553
554
  t.ctxPercent ?? null,
554
555
  t.pressureBand ?? null,
555
556
  t.model ?? null,
557
+ t.epochId ?? null,
556
558
  );
557
559
  const row = this.db
558
560
  .prepare(