pi-mega-compact 0.20.23 → 0.20.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,6 +50,8 @@ export const SETTINGS = [
50
50
  boolFlag("MEGACOMPACT_MEMORY_GRAPH", "Memory Graph", "Dashboard-oriented memory graph traversal", true),
51
51
  boolFlag("MEGACOMPACT_HYDE", "HyDE", "Generate hypothetical answer via LLM, embed it, RRF-fuse", true, true),
52
52
  boolFlag("MEGACOMPACT_NEW_UI", "New Dashboard UI", "Tailwind + shadcn visual design (sidebar, glass panels)", true),
53
+ boolDirect("MEGACOMPACT_MESSAGE_SEPARATION", "Message Separation (P2)", "PLAN_V2: split conversation thread from tool results to grow stable cache prefix", false),
54
+ boolDirect("MEGACOMPACT_CACHE_STRIPING", "Cache Striping (P3)", "PLAN_V2: order stable context by stability score so durable chunks lead the prompt", false),
53
55
  ],
54
56
  },
55
57
  {
@@ -10,12 +10,23 @@ export function cacheStripe(event, runtime, config) {
10
10
  const currentTurn = event.turnIndex;
11
11
  const embedder = defaultEmbedder();
12
12
  // (a) Extract text from the assistant's response for topic embedding.
13
+ // pi's assistant content may be a plain string OR an array of parts.
14
+ // Discriminate by type and only reach into `.text` after confirming the
15
+ // part is an object carrying a string `.text` — never assume shape.
13
16
  let textToEmbed = "";
14
17
  const msg = event.message;
15
- if (msg.role === "assistant" && Array.isArray(msg.content)) {
16
- for (const part of msg.content) {
17
- if ("text" in part && typeof part.text === "string") {
18
- textToEmbed += part.text + " ";
18
+ if (msg.role === "assistant") {
19
+ const c = msg.content;
20
+ if (typeof c === "string") {
21
+ textToEmbed = c;
22
+ }
23
+ else if (Array.isArray(c)) {
24
+ for (const part of c) {
25
+ if (part && typeof part === "object" && "text" in part) {
26
+ const t = part.text;
27
+ if (typeof t === "string")
28
+ textToEmbed += t + " ";
29
+ }
19
30
  }
20
31
  }
21
32
  }
@@ -23,9 +23,12 @@ import { VC6A_ENABLED, VC6C_ENABLED } from "../../../src/config/vector-cortex.js
23
23
  * topic model, and dedup — gated on dbMirror. Non-fatal end-to-end.
24
24
  */
25
25
  export async function persistEpochAndMaintain(runtime, config, ran) {
26
- // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
27
- // This makes the cache key stable across identical compactions.
28
- if (config.dbMirror) {
26
+ // PLAN_V2 cache-striping epoch row. buildCacheOptimizedPrompt's stripe
27
+ // lookup reads the most-recent checkpoint_epochs row, so emit it whenever
28
+ // PLAN_V2 flags are live. Separate from the legacy dbMirror block below so
29
+ // PLAN_V2 can run with dbMirror OFF (flag-on is a genuine additive; flag-OFF
30
+ // = byte-identical to the predecessor, since this block no-ops then).
31
+ if (config.dbMirror || config.messageSeparation || config.cacheStriping) {
29
32
  try {
30
33
  const db = openStore(runtime.currentStateDir);
31
34
  const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
@@ -40,11 +43,27 @@ export async function persistEpochAndMaintain(runtime, config, ran) {
40
43
  createdAt: Date.now(),
41
44
  };
42
45
  writeCheckpointEpoch(db, epoch);
46
+ }
47
+ catch (e) {
48
+ runtime.logger.warn("planv2-epoch-fail", { error: String(e) });
49
+ }
50
+ }
51
+ // S27 DB-mirror downstream: stamp turns, rebuild wiki, seed topics, dedup.
52
+ // Remains on `config.dbMirror` only — these are DB-mirror maintenance and
53
+ // must NOT run for PLAN_V2-only configurations.
54
+ if (config.dbMirror) {
55
+ try {
56
+ const db = openStore(runtime.currentStateDir);
57
+ const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
58
+ // epoch + writeCheckpointEpoch moved to the PLAN_V2 block above (it
59
+ // gates the wider set of flags); reuse epochIdFor(cpId) here so the
60
+ // stamps below land in the same epoch row.
61
+ const epochId = epochIdFor(cpId);
43
62
  // S50B: link this session's turns to the epoch that just compacted
44
63
  // them (compression-by-conversation-epoch metrics). Isolated-store
45
64
  // only; best-effort + non-fatal.
46
65
  try {
47
- stampTurnsEpochFor(config, runtime.rt.sessionId, epoch.epochId, runtime.currentStateDir);
66
+ stampTurnsEpochFor(config, runtime.rt.sessionId, epochId, runtime.currentStateDir);
48
67
  }
49
68
  catch {
50
69
  /* non-fatal: epoch stamping never breaks compaction */
@@ -3,6 +3,21 @@ import { appendMirrorMessages } from "../mirror-append.js";
3
3
  import { appendMessagesToLedger } from "../../mega-runtime/vector-cortex-ledger.js";
4
4
  import { epochIdFor } from "../../../src/mirror/epoch.js";
5
5
  import { messageContentText } from "./messageText.js";
6
+ /**
7
+ * Best-effort tool_call_id for the tool_results insert. The toolResult variant
8
+ * carries a top-level toolCallId (read via an `unknown`-narrowed cast — never
9
+ * reach into `.content`, which is variant-specific and requires narrowing).
10
+ * bashExecution has no toolCallId, so fall back to a stable synthetic id keyed
11
+ * on (turn, index) to satisfy the NOT NULL column. No `any` (PREVENT-011).
12
+ */
13
+ function toolCallIdOf(m, fallback) {
14
+ if (m.role === "toolResult") {
15
+ const id = m.toolCallId;
16
+ if (typeof id === "string" && id.length > 0)
17
+ return id;
18
+ }
19
+ return fallback;
20
+ }
6
21
  /**
7
22
  * Append incoming messages to the DB mirror (raw_transcript + thread/tool
8
23
  * tables) and the v2 ledger. Gated on config.dbMirror for the mirror; the VC1B
@@ -26,22 +41,35 @@ export function appendMirrorAndLedger(runtime, config, messages) {
26
41
  // separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
27
42
  // tailResult) is sufficient for the prompt-construction path;
28
43
  // these DB writes persist the split for post-hoc analysis, dashboard
29
- // queries, and future readers. Non-fatal failure here never breaks
30
- // the agent loop (PREVENT-PI-004: zero network, local SQLite only).
31
- {
44
+ // queries, and future readers. Gated on (messageSeparation ||
45
+ // cacheStriping) so flag-OFF remains byte-identical to the
46
+ // predecessor — when both flags are OFF the live prompt is never
47
+ // separated, and growing these tables would be dead state.
48
+ // Non-fatal — failure here never breaks the agent loop
49
+ // (PREVENT-PI-004: zero network, local SQLite only).
50
+ if (config.messageSeparation || config.cacheStriping) {
32
51
  const sid = runtime.rt.sessionId;
33
52
  const turn = runtime.currentTurn;
34
53
  const now = Date.now();
35
54
  const threadStmt = db.prepare("INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
36
- const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
37
- for (const m of messages) {
55
+ // Schema (plan-v2.ts) is (conversation_id, tool_call_id,
56
+ // tool_result, turn_index, timestamp) — NOT role/content.
57
+ const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, tool_call_id, tool_result, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
58
+ const toolHas = db.prepare("SELECT 1 FROM tool_results WHERE conversation_id = ? AND turn_index = ? AND tool_call_id = ? AND tool_result = ? LIMIT 1");
59
+ const threadHas = db.prepare("SELECT 1 FROM conversation_thread WHERE conversation_id = ? AND turn_index = ? AND role = ? AND content = ? LIMIT 1");
60
+ for (const [idx, m] of messages.entries()) {
38
61
  const role = m.role;
39
62
  const content = messageContentText(m);
40
63
  if (role === "user" || role === "assistant") {
41
- threadStmt.run(sid, role, content, turn, now);
64
+ if (threadHas.get(sid, turn, role, content) == null) {
65
+ threadStmt.run(sid, role, content, turn, now);
66
+ }
42
67
  }
43
68
  else if (role === "toolResult" || role === "bashExecution") {
44
- toolStmt.run(sid, role, content, turn, now);
69
+ const toolCallId = toolCallIdOf(m, `bash:${turn}:${idx}`);
70
+ if (toolHas.get(sid, turn, toolCallId, content) == null) {
71
+ toolStmt.run(sid, toolCallId, content, turn, now);
72
+ }
45
73
  }
46
74
  }
47
75
  }
@@ -1,6 +1,8 @@
1
1
  import { stagedForTail, withRecallTail } from "../recall-tail.js";
2
2
  import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "../separated-prompt.js";
3
3
  import { messageContentText } from "./messageText.js";
4
+ import { computeContentDigest } from "../../../src/dedup/digest.js";
5
+ const lastTurnPrefix = new WeakMap();
4
6
  /**
5
7
  * Build the tail injection closure. Returns undefined when nothing is staged
6
8
  * (or the flags are OFF) so the caller falls through to its normal return.
@@ -25,20 +27,51 @@ export function buildTailResult(runtime, config, messages) {
25
27
  else if (config.messageSeparation) {
26
28
  result = buildSeparatedPrompt(result);
27
29
  }
28
- // P2.5: log prefix stability (fire-and-forget, non-fatal).
29
- // tailResult is sync, so use .then().catch() on the dynamic import.
30
+ // P2.5: log cross-turn stable-prefix length (cache-hit proxy). After the
31
+ // prompt is built (separated or cache-optimized), fingerprint each leading
32
+ // message and count how many are byte-identical to the previous turn's
33
+ // prompt, in order. A high stablePrefix = the provider KV-cache prefix is
34
+ // re-used (cache hit). Fire-and-forget + non-fatal.
35
+ //
36
+ // tailResult may run several times per turn (gate return / replay /
37
+ // debounce / live-trim), so the sessionId+turn guard measures once per
38
+ // turn — compare against the PREVIOUS turn's stored footprint, then store
39
+ // this turn's for the next comparison. Cross-session compares are skipped
40
+ // via the stored sessionId check (belt-and-suspenders; the WeakMap entry
41
+ // is scoped to this runtime, which owns the session).
30
42
  if (result.length > 1) {
31
- import("../../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
32
- const stableScore = computeStabilityScore({ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 }, result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })));
33
- runtime.logger.info("prefix_stability", {
34
- stableScore: Number.isFinite(stableScore) ? stableScore : 0,
35
- prefixMessages: result.length,
36
- separation: config.messageSeparation ? "v2" : "off",
37
- striping: config.cacheStriping ? "v3" : "off",
38
- });
39
- }).catch(() => {
43
+ try {
44
+ const sessionId = runtime.rt.sessionId;
45
+ const turn = runtime.currentTurn;
46
+ const prev = lastTurnPrefix.get(runtime);
47
+ const isNewTurn = !prev || prev.sessionId !== sessionId || prev.turn !== turn;
48
+ if (isNewTurn) {
49
+ const fingerprints = result.map((m) =>
50
+ // Role prefix disambiguates identical text across variants
51
+ // (roles are a fixed enum — no realistic fingerprint collision).
52
+ computeContentDigest(`${m.role}|${messageContentText(m)}`).contentHash);
53
+ let stablePrefix = 0;
54
+ if (prev && prev.sessionId === sessionId) {
55
+ for (let i = 0; i < fingerprints.length; i++) {
56
+ if (i >= prev.fingerprints.length ||
57
+ prev.fingerprints[i] !== fingerprints[i]) {
58
+ break;
59
+ }
60
+ stablePrefix++;
61
+ }
62
+ }
63
+ lastTurnPrefix.set(runtime, { sessionId, turn, fingerprints });
64
+ runtime.logger.info("prefix_stability", {
65
+ stablePrefix,
66
+ totalMessages: result.length,
67
+ separation: config.messageSeparation ? "v2" : "off",
68
+ striping: config.cacheStriping ? "v3" : "off",
69
+ });
70
+ }
71
+ }
72
+ catch {
40
73
  // Non-fatal: stability logging is best-effort.
41
- });
74
+ }
42
75
  }
43
76
  return { messages: result };
44
77
  };
@@ -1,240 +1,82 @@
1
1
  /**
2
- * cache-stripe-impl.ts — Vector-Aware Cache Striping implementation.
2
+ * cache-stripe-impl.ts — Vector-Aware Cache Striping (PLAN_V2 Phase 3).
3
3
  *
4
- * Computes a stability score for each context chunk and assigns it to a
5
- * cache stripe / prompt-cache layer. Stable chunks (high recency + frequency +
6
- * semantic density) go to early layers (cached prefix); volatile chunks append
7
- * at Layer 4 (tail). Runs entirely offline no network, no LLM (PREVENT-PI-004).
4
+ * Owns refreshStripeAssignments: the DB-touching write path that scores each
5
+ * context chunk and UPSERTs its stripe row into cache_stripes. All pure math
6
+ * and scoring types live in cache-stripe-score.ts (extracted via the
7
+ * delegate-shell pattern to keep this file under the 300-line src/ soft
8
+ * limit). cache-stripe.ts re-exports the public surface.
8
9
  *
9
- * The stability score is a weighted composite:
10
- * stability = 0.5 * semanticSimilarity + 0.3 * recency + 0.2 * frequency
11
- *
12
- * - semanticSimilarity: cosine similarity of the chunk's embedding against the
13
- * running session embedding (from TrigramEmbedder). High similarity means the
14
- * chunk is topically relevant to current work.
15
- * - recency: how recently the chunk appeared (normalized to 0.0-1.0 across all
16
- * chunks in the epoch). Recent chunks are more likely to benefit from caching.
17
- * - frequency: how often the chunk's content has been referenced (0.0-1.0,
18
- * estimated from a simple access counter stored alongside).
19
- *
20
- * Reassignment happens at epoch boundaries via refreshStripeAssignments.
21
- * All SQL is parameterized (PREVENT-002). No pi runtime types are imported,
22
- * keeping this module pi-agnostic.
10
+ * Runs entirely offline no network, no LLM (PREVENT-PI-004). All SQL is
11
+ * parameterized (PREVENT-002). No pi runtime types are imported, keeping this
12
+ * module pi-agnostic.
23
13
  */
24
14
  import { randomBytes } from "node:crypto";
25
15
  import { openStore, withTx } from "./store/sqlite/utils.js";
26
- // ─── Constants ───────────────────────────────────────────────────────────────
27
- /** Semantic similarity weight in the composite score. */
28
- const WEIGHT_SEMANTIC = 0.5;
29
- /** Recency weight in the composite score. */
30
- const WEIGHT_RECENCY = 0.3;
31
- /** Frequency weight in the composite score. */
32
- const WEIGHT_FREQUENCY = 0.2;
33
- /** Stripes a chunk lands in based on its stability score. Thresholds define
34
- * the boundary between adjacent layers. */
35
- const STRIPE_THRESHOLDS = [
36
- { minStability: 0.90, stripe: 0 },
37
- { minStability: 0.70, stripe: 1 },
38
- { minStability: 0.50, stripe: 2 },
39
- { minStability: 0.30, stripe: 3 },
40
- { minStability: -Infinity, stripe: 4 },
41
- ];
42
- // ─── Embedding helpers (no external dep) ─────────────────────────────────────
43
- /**
44
- * FNV-1a 32-bit hash for the content-based embedding fallback. The production
45
- * path uses TrigramEmbedder from embedder.ts but we keep a self-contained hash
46
- * for the case where no embedder is passed in.
47
- */
48
- function fnv1a(text) {
49
- let hash = 0x811c9dc5;
50
- for (let i = 0; i < text.length; i++) {
51
- hash ^= text.charCodeAt(i);
52
- hash = Math.imul(hash, 0x01000193);
53
- }
54
- return (hash >>> 0) / 0x100000000;
55
- }
56
- /**
57
- * Produce a crude 128-dim pseudorandom embedding from text using hashed n-gram
58
- * bins. Matches the approach in TrigramEmbedder._embedRaw conceptually. Used
59
- * only as a fallback / test path; the caller should prefer TrigramEmbedder.
60
- */
61
- function fallbackEmbed(text) {
62
- const dim = 128;
63
- const vec = new Array(dim).fill(0);
64
- const norm = text.toLowerCase().replace(/\s+/g, " ");
65
- if (norm.length === 0)
66
- return vec;
67
- vec[Math.floor(fnv1a(norm) * dim)] += 1;
68
- for (const word of norm.split(" ")) {
69
- if (word.length === 0)
70
- continue;
71
- vec[Math.floor(fnv1a(word) * dim)] += 0.5;
72
- for (let i = 0; i < Math.max(1, word.length - 1); i++) {
73
- const trigram = word.slice(i, i + 3);
74
- if (trigram.length === 3) {
75
- vec[Math.floor(fnv1a(trigram) * dim)] += 0.25;
76
- }
77
- }
78
- }
79
- return l2Normalize(vec);
80
- }
81
- function l2Normalize(v) {
82
- let sumSq = 0;
83
- for (let i = 0; i < v.length; i++)
84
- sumSq += v[i] * v[i];
85
- if (sumSq === 0)
86
- return v;
87
- const norm = Math.sqrt(sumSq);
88
- for (let i = 0; i < v.length; i++)
89
- v[i] /= norm;
90
- return v;
91
- }
92
- /** Compute cosine similarity between two vectors of equal length. */
93
- function cosineSimilarity(a, b) {
94
- if (a.length !== b.length || a.length === 0)
95
- return 0;
96
- let dot = 0;
97
- let na = 0;
98
- let nb = 0;
99
- for (let i = 0; i < a.length; i++) {
100
- dot += a[i] * b[i];
101
- na += a[i] * a[i];
102
- nb += b[i] * b[i];
103
- }
104
- const denom = Math.sqrt(na) * Math.sqrt(nb);
105
- return denom === 0 ? 0 : dot / denom;
106
- }
107
- // ─── Stability Scoring ───────────────────────────────────────────────────────
108
- /**
109
- * Compute a composite stability score for a single chunk.
110
- *
111
- * @param chunk The chunk metadata + content to score.
112
- * @param allChunks All chunks in this epoch (used to compute relative recency).
113
- * @param embedder Optional embedder instance. If omitted, uses the
114
- * self-contained fallback (128-dim hashed n-gram).
115
- * @param sessionEmbed Pre-computed embedding for the current session (the
116
- * "query" vector). If omitted, computed on the fly from
117
- * the chunk content alone, which degrades semantic scoring
118
- * to a self-similarity baseline.
119
- * @returns A number in [0.0, 1.0] where 1.0 = most stable.
120
- */
121
- export function computeStabilityScore(chunk, allChunks, embedder, sessionEmbed) {
122
- // ── Semantic similarity (0.5 weight) ────────────────────────────────────
123
- const emb = embedder
124
- ? embedder.embed(chunk.content)
125
- : fallbackEmbed(chunk.content);
126
- // If no session embedding is provided, use the chunk's own embedding as
127
- // a self-similarity — this produces a baseline score based on content
128
- // density (chunks with more meaningful content get higher internal
129
- // similarity). Real deployments should pass the session embedding.
130
- const sem = cosineSimilarity(emb, sessionEmbed ?? emb);
131
- const semanticScore = isNaN(sem) ? 0 : sem;
132
- // ── Recency (0.3 weight) ────────────────────────────────────────────────
133
- // Relative recency: lastAccessedAt of this chunk vs. min/max across epoch.
134
- // Falls back to 0.5 if there's only one chunk or no timestamp data.
135
- let recencyScore = 0.5;
136
- const accessed = allChunks
137
- .map((c) => c.lastAccessedAt)
138
- .filter((t) => t > 0);
139
- if (accessed.length > 1) {
140
- const minT = Math.min(...accessed);
141
- const maxT = Math.max(...accessed);
142
- const range = maxT - minT;
143
- if (range > 0) {
144
- recencyScore = (chunk.lastAccessedAt - minT) / range;
145
- }
146
- else {
147
- recencyScore = 1.0;
148
- }
149
- }
150
- // ── Frequency (0.2 weight) ──────────────────────────────────────────────
151
- // Access count relative to the max across the epoch.
152
- const counts = allChunks.map((c) => c.accessCount);
153
- const maxCount = Math.max(...counts, 1);
154
- const freqScore = maxCount > 0 ? chunk.accessCount / maxCount : 0;
155
- // ── Composite ───────────────────────────────────────────────────────────
156
- const stability = WEIGHT_SEMANTIC * semanticScore +
157
- WEIGHT_RECENCY * recencyScore +
158
- WEIGHT_FREQUENCY * freqScore;
159
- // Clamp to [0.0, 1.0] as a safety net.
160
- return Math.max(0, Math.min(1, stability));
161
- }
162
- /**
163
- * Determine the cache stripe (layer) for a given stability score.
164
- *
165
- * @param stability Composite stability score in [0.0, 1.0].
166
- * @returns Stripe number 0-4.
167
- */
168
- export function stabilityToStripe(stability) {
169
- for (const t of STRIPE_THRESHOLDS) {
170
- if (stability >= t.minStability)
171
- return t.stripe;
172
- }
173
- return 4;
174
- }
16
+ import { computeStabilityScore, stabilityToStripe, fallbackEmbed, l2Normalize, } from "./cache-stripe-score.js";
17
+ export { computeStabilityScore, stabilityToStripe, fallbackEmbed, l2Normalize, } from "./cache-stripe-score.js";
175
18
  // ─── Stripe Reassignment ─────────────────────────────────────────────────────
176
19
  /**
177
- * Refresh stripe assignments for all chunks in the current epoch.
20
+ * Refresh stripe assignments for all chunks in an epoch.
178
21
  *
179
22
  * Steps:
180
- * 1. Read all context_chunks from the SQLite store that belong to the
181
- * current epoch (or all chunks if no epoch filter).
182
- * 2. For each chunk, compute a stability score via computeStabilityScore.
183
- * 3. Map stability -> stripe via stabilityToStripe.
184
- * 4. UPSERT into cache_stripes.
185
- * 5. (Stale entries for this epoch are implicitly overwritten by the UPSERT.)
186
- *
187
- * The optional embedder parameter allows injecting the production
188
- * TrigramEmbedder. If omitted, the fallback hashed n-gram embedder is used
189
- * (works offline in all scenarios).
23
+ * 1. Resolve target epochId: explicit string use it; '' no epoch filter
24
+ * (assign all chunks); undefined look up the most recently committed
25
+ * checkpoint_epochs row so the stripes we write are visible to the
26
+ * buildCacheOptimizedPrompt reader (`ORDER BY created_at DESC LIMIT 1`).
27
+ * If no epoch exists yet (pre-first-compaction), fall back to a random id
28
+ * the write succeeds and the rows are simply never read until an epoch
29
+ * lands.
30
+ * 2. Read context_chunks (chunk_id = c.id, NOT rowid — the read path joins
31
+ * on the TEXT id).
32
+ * 3. Score each chunk via computeStabilityScore (mean-pool embeddings for a
33
+ * session-level semantic "query" vector).
34
+ * 4. Atomic UPSERT into cache_stripes.
190
35
  *
191
- * Non-fatal: failures are logged via a provided logger callback and never
192
- * thrown. Returns the count of chunks reassigned.
193
- *
194
- * @param store An open SQLite DatabaseSync handle (or a stateDir string
195
- * to open lazily). Accepts either to match the caller's
196
- * convenience. When a string is passed, opens the store for
197
- * this call only (does not cache the connection).
198
- * @param epochId The epoch to reassign. If omitted, generates a new epoch
199
- * ID (random hex). Pass '' to reassign all chunks without
200
- * filtering by epoch.
201
- * @param embedder Optional TrigramEmbedder instance. When provided, uses it
202
- * for semantic similarity; otherwise uses the fallback.
203
- * @param logFn Optional logging callback (defaults to no-op).
204
- * @returns The number of chunks that were reassigned.
36
+ * Non-fatal: failures are logged via the provided logger and never thrown.
37
+ * Returns the count of chunks reassigned (0 on error).
205
38
  */
206
39
  export function refreshStripeAssignments(store, epochId, embedder, logFn) {
207
40
  const db = typeof store === "string" ? openStore(store) : store;
208
41
  const log = logFn ?? (() => { });
209
- const actualEpochId = epochId ?? nextEpochId();
42
+ let actualEpochId;
43
+ if (epochId !== undefined) {
44
+ actualEpochId = epochId;
45
+ }
46
+ else {
47
+ try {
48
+ const latest = db
49
+ .prepare(`SELECT epoch_id FROM checkpoint_epochs ORDER BY created_at DESC LIMIT 1`)
50
+ .get();
51
+ actualEpochId = latest?.epoch_id ?? nextEpochId();
52
+ }
53
+ catch {
54
+ actualEpochId = nextEpochId();
55
+ }
56
+ }
210
57
  const now = Math.floor(Date.now() / 1000);
211
58
  try {
212
- // 1. Read all relevant context_chunks, using the summary as text content.
213
- // The summary field holds the compressed checkpoint content; for fresh
214
- // chunks that have no summary yet, fall back to normalized_text or
215
- // concatenated key_decisions.
59
+ // cache_stripes has no access_count / last_accessed_at columns (schema:
60
+ // chunk_id/stripe/stability/assigned_at/epoch_id) querying them throws.
61
+ // Score freshness/frequency to 0 here; stability derives from content +
62
+ // semantic similarity only until an access-tracking column is added.
216
63
  const rows = db
217
- .prepare(`SELECT c.rowid AS chunk_id,
218
- COALESCE(c.summary, c.normalized_text, c.key_decisions, '') AS content,
219
- COALESCE(s.access_count, 0) AS access_count,
220
- COALESCE(s.last_accessed_at, 0) AS last_accessed_at
64
+ .prepare(`SELECT c.id AS chunk_id,
65
+ COALESCE(c.summary, c.normalized_text, c.key_decisions, '') AS content
221
66
  FROM context_chunks c
222
- LEFT JOIN cache_stripes s ON s.chunk_id = CAST(c.rowid AS TEXT)
67
+ LEFT JOIN cache_stripes s ON s.chunk_id = c.id
223
68
  WHERE (? = '' OR s.epoch_id = ? OR s.epoch_id IS NULL)`)
224
69
  .all(actualEpochId, actualEpochId);
225
70
  if (rows.length === 0) {
226
71
  log("cache-stripe: no chunks to reassign");
227
72
  return 0;
228
73
  }
229
- // Build the allChunks array for relative scoring.
230
74
  const allChunks = rows.map((r) => ({
231
- chunkId: String(r.chunk_id),
75
+ chunkId: r.chunk_id,
232
76
  content: r.content,
233
- accessCount: r.access_count,
234
- lastAccessedAt: r.last_accessed_at,
77
+ accessCount: 0,
78
+ lastAccessedAt: 0,
235
79
  }));
236
- // Compute a session embedding (mean of all chunk embeddings) for semantic
237
- // similarity comparison.
238
80
  let sessionEmbed;
239
81
  try {
240
82
  const dim = embedder ? embedder.embed("").length : 128;
@@ -257,14 +99,12 @@ export function refreshStripeAssignments(store, epochId, embedder, logFn) {
257
99
  catch {
258
100
  log("cache-stripe: session embedding failed, skipping semantic weight");
259
101
  }
260
- // 2. Compute stability for each chunk.
261
102
  const results = [];
262
103
  for (const chunk of allChunks) {
263
104
  const stability = computeStabilityScore(chunk, allChunks, embedder, sessionEmbed);
264
105
  const stripe = stabilityToStripe(stability);
265
106
  results.push({ chunkId: chunk.chunkId, stripe, stability });
266
107
  }
267
- // 3. UPSERT into cache_stripes using a savepoint for atomicity.
268
108
  const upsert = db.prepare(`INSERT OR REPLACE INTO cache_stripes(chunk_id, stripe, stability, assigned_at, epoch_id)
269
109
  VALUES (?, ?, ?, ?, ?)`);
270
110
  withTx(db, () => {
@@ -280,9 +120,7 @@ export function refreshStripeAssignments(store, epochId, embedder, logFn) {
280
120
  return 0;
281
121
  }
282
122
  }
283
- /**
284
- * Generate a random epoch ID (16 hex chars) for tokenizing stripe cohorts.
285
- */
123
+ /** Generate a random epoch id (16 hex chars) for tokenizing stripe cohorts. */
286
124
  function nextEpochId() {
287
125
  return randomBytes(8).toString("hex");
288
126
  }