synabun 1.3.0 → 2.0.0

package/mcp-server/src/services/memory-retrieval.ts

@@ -0,0 +1,349 @@
+import { performance } from 'node:perf_hooks';
+import { createHash } from 'node:crypto';
+import { getDb, memoryVectors, chunkVectors, rowToPayload, rowToSessionChunkPayload } from './sqlite.js';
+import { VectorCache } from './vector-cache.js';
+import { judgeHistoricalQuery, judgeRelevance, orderByJudgedScore, RELEVANCE_CANDIDATE_CLIP, type JudgeContext } from './memory-judgments.js';
+import { generateEmbedding, EMBEDDING_VERSION } from './local-embeddings.js';
+import { projectAliases } from '../config.js';
+import { judgedStale } from './edit-stale.js';
+
+export interface RecallOptions {
+  query: string; project?: string; currentProject?: string; category?: string; tags?: string[];
+  tag_match?: 'any' | 'all'; limit?: number; min_score?: number; min_importance?: number;
+  include_sessions?: boolean; recency_boost?: boolean; after?: string; before?: string;
+  scope?: 'all' | 'project'; engine?: 'legacy' | 'hybrid';
+  // Internal automatic-injection gate; explicit recall keeps keyword-only hits.
+  min_semantic_score?: number;
+  // Judge the shortlist after fusion. Opt-in: it cannot start until retrieval
+  // has finished, so unlike the superseded judgment it is not free wall clock.
+  // Off for automatic injection, which answers inside a fixed hook budget.
+  rerank?: boolean;
+  // false: no judgment of any kind for this call (a session that opted out).
+  judge?: boolean;
+  // Attribution for the judgments this call makes (origin, session, project).
+  judgeContext?: JudgeContext;
+}
+export interface RecallHit {
+  id: string; type: 'memory' | 'session'; content: string; project: string; category: string;
+  source: string; created_at: string; score: number; revision: number; flags: string[];
+  payload: Record<string, any>; reasons: string[];
+  semantic_score?: number; passage_score?: number; keyword_coverage?: number; fusion_score?: number;
+}
+export interface RecallResult { results: RecallHit[]; degraded: string[]; engine: string; elapsed_ms: number; timings?: Record<string,number> }
+const passages = new VectorCache(getDb, 'SELECT id,vector FROM memory_passages', 'SELECT vector FROM memory_passages WHERE id=?', 'memory_passages');
+const metrics = { calls: 0, degraded: 0, total_ms: 0, last_ms: 0 };
+const recentTimes: number[]=[];
+const filterCache=new Map<string,Set<string>>();
+let filterConnection: ReturnType<typeof getDb> | null=null;
+let filterEpoch=-1;
+function memoryIds(sql:string,params:(string|number)[]) {
+  const d=getDb();
+  const epoch=Number(d.prepare("SELECT COALESCE(MAX(seq),0) AS n FROM memory_changes WHERE entity='memory_filters'").get()?.n || 0);
+  if(d!==filterConnection || epoch!==filterEpoch){filterCache.clear();filterConnection=d;filterEpoch=epoch;}
+  const key=JSON.stringify([sql,params]);
+  let ids=filterCache.get(key);
+  if(!ids) {
+    ids=new Set(d.prepare(`SELECT m.id FROM memories m WHERE ${sql}`).all(...params).map(r=>String(r.id)));
+    filterCache.set(key,ids);
+    let total=0;for(const value of filterCache.values())total+=value.size;
+    while(filterCache.size>32 || (total>100000 && filterCache.size>1)) {
+      const oldest=filterCache.keys().next().value!;total-=filterCache.get(oldest)!.size;filterCache.delete(oldest);
+    }
+  }
+  return ids;
+}
+export function retrievalMetrics() {
+  const sorted=[...recentTimes].sort((a,b)=>a-b);
+  return { ...metrics, p50_ms:sorted[Math.floor(sorted.length*.5)] ?? 0, p95_ms:sorted[Math.floor(sorted.length*.95)] ?? 0 };
+}
+export function resetPassageCache() { passages.invalidate(); }
+export function hybridEnabled(): boolean {
+  if (process.env.SYNABUN_MEMORY_ENGINE) return process.env.SYNABUN_MEMORY_ENGINE === 'hybrid';
+  const configured=(getDb().prepare("SELECT value FROM kv_config WHERE key='memory_engine'").get() as {value:string} | undefined)?.value;
+  // Enabled after the checked-in relevance, latency and compatibility gates.
+  // An explicit legacy setting remains an immediate retrieval rollback.
+  return configured ? configured === 'hybrid' : true;
+}
+const stopwords = new Set('the a an is are was were to of for in on and or with my our your we i how what when about do did does this that it at as by from please memory memories synabun'.split(' '));
+const conversationalWords = new Set('there here better same something anything can could would should me you us some any also just maybe really now then than'.split(' '));
+export function queryTerms(query: string): string[] {
+  const tokens = query.toLowerCase().match(/[\p{L}\p{N}_]+(?:[-./:][\p{L}\p{N}_]+)*/gu) || [];
+  const terms = [...new Set(tokens.filter(t => !stopwords.has(t) && t.length > 1))];
+  const specific = terms.filter(t => !conversationalWords.has(t));
+  return (specific.length ? specific : terms).slice(0, 32);
+}
+
+function filterSql(o: RecallOptions, session: boolean) {
+  const clauses = session ? ['NOT EXISTS(SELECT 1 FROM memories hidden WHERE hidden.id=m.id AND hidden.trashed_at IS NOT NULL)'] : ['m.trashed_at IS NULL'];
+  const params: (string | number)[] = [];
+  const project = o.project || (o.scope === 'project' ? o.currentProject : undefined);
+  if (o.scope === 'project' && (!project || (!o.project && project === 'global'))) throw new Error('Project scope requires a project or caller project context.');
+  if (project) {
+    const aliases = projectAliases(project);
+    clauses.push(`lower(m.project) IN (${aliases.map(() => '?').join(',')})`);
+    params.push(...aliases.map(p => p.toLowerCase()));
+  }
+  if (o.category) { clauses.push(session ? (o.category === 'conversations' ? '1=1' : '0=1') : 'm.category=?'); if (!session) params.push(o.category); }
+  if (o.tags?.length) {
+    if (session) clauses.push('0=1'); // session tools are not memory tags
+    else {
+      clauses.push('(' + o.tags.map(tag => { params.push(tag); return 'EXISTS(SELECT 1 FROM json_each(m.tags) WHERE value=?)'; }).join(o.tag_match === 'all' ? ' AND ' : ' OR ') + ')');
+    }
+  }
+  if (o.min_importance) { clauses.push(session ? '0=1' : 'm.importance>=?'); if (!session) params.push(o.min_importance); }
+  if (o.after) { clauses.push(`${session ? 'm.start_timestamp' : 'm.created_at'}>=?`); params.push(o.after); }
+  if (o.before) { clauses.push(`${session ? 'm.start_timestamp' : 'm.created_at'}<=?`); params.push(o.before); }
+  return { sql: clauses.join(' AND '), params };
+}
+
+function hydrate(row: Record<string, any>, session: boolean, score = 0): RecallHit {
+  const p = session ? rowToSessionChunkPayload(row) : rowToPayload(row);
+  return { id: row.id, type: session ? 'session' : 'memory', content: p.content, project: p.project,
+    category: session ? 'conversations' : row.category, source: session ? 'session' : row.source,
+    created_at: session ? row.start_timestamp : row.created_at, score,
+    revision: session ? 0 : Number(row.revision || 1), flags: [], payload: p, reasons: [] };
+}
+
+/** Both adapters call this engine; formatting remains transport-specific. */
+export async function retrieveMemory(o: RecallOptions, embed = generateEmbedding): Promise<RecallResult> {
+  const start = performance.now();
+  const timings: Record<string,number>={};
+  let stage=start;
+  const mark=(name:string)=>{const now=performance.now();timings[name]=now-stage;stage=now;};
+  const d = getDb();
+  const engine = o.engine || (hybridEnabled() ? 'hybrid' : 'legacy');
+  const hybrid = engine === 'hybrid';
+  const limit = Math.max(1, Math.min(50, o.limit ?? 5));
+  const k = hybrid ? 50 : limit * 2;
+  const min = o.min_score ?? 0.3;
+  const terms = queryTerms(o.query);
+  const degraded: string[] = [];
+  const found = new Map<string, RecallHit>();
+  const ranks = new Map<string, number>();
+  const memoryFilter = filterSql(o, false);
+  const sessionFilter = filterSql(o, true);
+  const wantedSessions = o.include_sessions ?? (/yesterday|last |previous|recent|session|conversation|ontem|sessão|conversa/i.test(o.query));
+  let vector: number[] | null = null;
+  // UUID reads bypass inference entirely.
+  const uuid = o.query.trim().match(/^[0-9a-f]{8}-[0-9a-f-]{27}$/i)?.[0];
+  if (uuid) {
+    const row = d.prepare(`SELECT m.* FROM memories m WHERE m.id=? AND ${memoryFilter.sql}`).get(uuid, ...memoryFilter.params);
+    if (row) { const h = hydrate(row, false, 1); h.reasons = ['exact UUID']; return {results:[h],degraded,engine,elapsed_ms:performance.now()-start}; }
+  }
+  const pendingEmbedding=embed(o.query).catch(()=>null);
+
+  function add(hit: RecallHit, rank: number, reason: string) {
+    const key = `${hit.type}:${hit.id}`;
+    const previous = found.get(key);
+    const channel = ({semantic:'semantic_score',passage:'passage_score',keyword:'keyword_coverage'} as const)[reason as 'semantic'|'passage'|'keyword'];
+    if (channel) {
+      hit[channel] = hit.score;
+      if (previous) previous[channel] = Math.max(previous[channel] ?? -Infinity, hit.score);
+    }
+    // A long document can match many passages; it gets one vote per channel.
+    if (previous?.reasons.includes(reason)) { previous.score=Math.max(previous.score,hit.score); return; }
+    if (previous) { previous.score = Math.max(previous.score, hit.score); previous.reasons.push(reason); }
+    else { hit.reasons.push(reason); found.set(key, hit); }
+    ranks.set(key, (ranks.get(key) || 0) + 1 / (60 + rank));
+  }
+  function search(session: boolean, lexical: boolean) {
+    const table = session ? 'session_chunks' : 'memories';
+    const filter = session ? sessionFilter : memoryFilter;
+    if (lexical) {
+      if (!terms.length) return;
+      const fts = session ? 'chunk_search' : 'memory_search';
+      try {
+        const rows = d.prepare(`SELECT m.* FROM ${fts} f JOIN ${table} m ON m.id=f.id
+          WHERE ${fts} MATCH ? AND ${filter.sql} ORDER BY bm25(${fts}) LIMIT ?`)
+          .all(terms.map(t => `"${t.replace(/"/g,'""')}"`).join(' OR '), ...filter.params, k);
+        rows.forEach((row, i) => {
+          const h = hydrate(row, session);
+          const body = `${h.content} ${h.payload.related_files || h.payload.files_modified || ''}`.toLowerCase();
+          const covered = terms.filter(t => body.includes(t)).length / terms.length;
+          h.score = covered;
+          if (covered >= min) add(h, i + 1, 'keyword');
+        });
+      } catch (error) { degraded.push(`Keyword index unavailable (${session ? 'sessions' : 'memories'}).`); }
+    } else if (vector) {
+      const ids = session ? new Set((d.prepare(`SELECT m.id FROM ${table} m WHERE ${filter.sql}`).all(...filter.params)).map(r => String(r.id)))
+        : memoryIds(filter.sql,filter.params);
+      const hits = (session ? chunkVectors : memoryVectors).topK(vector, k, hybrid ? min : min * 0.5, ids);
+      const hydrateRow = d.prepare(`SELECT m.* FROM ${table} m WHERE m.id=? AND ${filter.sql}`);
+      hits.forEach((hit, i) => {
+        const row = hydrateRow.get(hit.id,...filter.params);
+        if (row) add(hydrate(row, session, hit.score), i + 1, 'semantic');
+      });
+    }
+  }
+  if(hybrid)search(false,true);
+  mark('keyword');
+  vector=await pendingEmbedding;
+  mark('embedding_wait');
+  if(!vector)degraded.push('Local embeddings unavailable; keyword retrieval used.');
+  search(false, false);
+  mark('document_vectors');
+  if (!hybrid && (!vector || found.size < limit)) search(false, true);
+  if (hybrid && vector && found.size) {
+    const parents=[...found.values()].filter(r=>r.type==='memory').map(r=>r.id).slice(0,100);
+    const ids = new Set(d.prepare(`SELECT p.id FROM memory_passages p INDEXED BY memory_passages_parent CROSS JOIN memories m ON m.id=p.memory_id
+      JOIN memory_metadata md ON md.memory_id=m.id WHERE p.memory_id IN (${parents.map(()=>'?').join(',')}) AND ${memoryFilter.sql} AND p.model=? AND p.content_hash=md.content_hash
+      AND NOT EXISTS(SELECT 1 FROM memory_jobs j WHERE j.entity_id=m.id AND j.kind='index' AND j.status!='complete')`)
+      .all(...parents,...memoryFilter.params, EMBEDDING_VERSION).map(r => String(r.id)));
+    mark('passage_filter');
+    const parentRow = d.prepare('SELECT m.* FROM memory_passages p JOIN memories m ON m.id=p.memory_id WHERE p.id=? AND m.trashed_at IS NULL');
+    for (const [i, hit] of passages.topK(vector, k, min, ids).entries()) {
+      const row = parentRow.get(hit.id);
+      if (row) add(hydrate(row, false, hit.score), i+1, 'passage');
+    }
+  }
+  mark('passage_vectors');
+  if (o.include_sessions !== false && (wantedSessions || found.size < 3)) { search(true,false); search(true,true); }
+  // Preserve actual document cosine even for a keyword-only candidate. Fusion
+  // rank is not a probability and must never substitute for semantic evidence.
+  if (vector) {
+    const ids = new Set([...found.values()].filter(h=>h.type==='memory').map(h=>h.id));
+    for (const h of memoryVectors.topK(vector, ids.size, -Infinity, ids)) {
+      const candidate = found.get(`memory:${h.id}`);
+      if (candidate) candidate.semantic_score = h.score;
+    }
+  }
+  const now = Date.now();
+  const aliases = o.currentProject ? projectAliases(o.currentProject).map(p=>p.toLowerCase()) : [];
+  const readRevision=d.prepare('SELECT MAX(revision) AS n FROM memory_revisions WHERE memory_id=?');
+  const readRelations=d.prepare(`SELECT r.*,m.id AS live FROM memory_relations r LEFT JOIN memories m ON m.id=CASE WHEN r.from_id=? THEN r.to_id ELSE r.from_id END
+    AND m.trashed_at IS NULL WHERE (r.from_id=? OR r.to_id=?) AND r.active=1`);
+  for (const [key, h] of found) {
+    const age = Math.max(0, (now - Date.parse(h.created_at)) / 86400000) || 0;
+    if (hybrid) {
+      h.score = ranks.get(key)!;
+      // Exact rare identifiers and dates take precedence over generic matches.
+      const identifiers = terms.filter(t => /[\d_./:]/.test(t));
+      if (identifiers.length && identifiers.every(t => h.content.toLowerCase().includes(t))) { h.score += 0.025; h.reasons.push('exact identifier'); }
+      if (o.recency_boost) h.score *= 1 + 0.15 * Math.pow(0.5, age / 14);
+      if (!o.project && aliases.includes(h.project.toLowerCase())) h.score *= 1.1;
+      h.fusion_score = h.score;
+    } else if (h.type === 'memory' && !h.reasons.includes('keyword')) {
+      const boost = Math.min(0.1, (h.payload.access_count || 0)*0.01);
+      h.score = o.recency_boost ? h.score*0.35 + Math.pow(0.5,age/14)*0.55+boost
+        : h.payload.importance >= 8 ? h.score : h.score*0.7+Math.pow(0.5,age/90)*0.2+boost;
+      if (!o.project && h.project === o.currentProject) h.score *= 1.2;
+    }
+    if (h.type === 'memory') {
+      const relations = readRelations.all(h.id,h.id,h.id);
+      for (const r of relations) {
+        if (!r.live) continue;
+        if (r.kind === 'supersedes' && r.to_id === h.id) h.flags.push('superseded');
+        if (r.kind === 'conflicts_with' || r.kind === 'possible_conflict') h.flags.push(String(r.kind));
+      }
+    }
+  }
+  const sorted = [...found.values()].filter(h => hybrid || h.score >= min).sort((a,b) =>
+    (!hybrid && a.type !== b.type ? (a.type === 'memory' ? -1 : 1) : b.score-a.score) || a.id.localeCompare(b.id));
+  // "Is this query about the past?" only decides whether superseded memories
+  // are shown, so it is asked only when one is among the candidates. Firing it
+  // on every recall (as the first version did, racing a fixed 1200 ms) made
+  // every recall wait on an answer nothing consulted: supersession is rare,
+  // and a third of those calls timed out from a cold connection anyway. When
+  // it is needed, the surface's own timeout bounds the wait and a miss falls
+  // back to the regex below.
+  const needsHistorical = hybrid && !o.before && o.judge !== false && sorted.some(h => h.flags.includes('superseded'));
+  const historical = needsHistorical ? await judgeHistoricalQuery(o.query, o.judgeContext) : null;
+  // Judging needs a shortlist wider than the answer, or there is nothing to reorder.
+  const collect = o.rerank && hybrid && o.judge !== false ? Math.max(limit, 10) : limit;
+  const results: RecallHit[] = [];
+  const seen = new Set<string>();
+  const byHash = new Map<string,string>();
+  for (const h of sorted) {
+    if (o.min_semantic_score !== undefined && Math.max(h.semantic_score ?? -Infinity, h.passage_score ?? -Infinity) < o.min_semantic_score) continue;
+    if(!hybrid && results.filter(r=>r.type===h.type).length >= (h.type==='memory' ? limit : Math.max(3,Math.floor(limit/2))))continue;
+    if(h.type==='session' && h.payload.dedup_memory_id && seen.has(h.payload.dedup_memory_id))continue;
+    // The judgment answers this in any language; the regex, which only ever
+    // spoke English and Portuguese, stays as the fallback when it is missing.
+    if (hybrid && h.flags.includes('superseded') && !o.before
+      && !(historical ?? /history|histor|previous|old|antes|anterior/i.test(o.query))) continue;
+    const hash = createHash('sha256').update(`${h.project.toLowerCase()}\0${h.category}\0${h.source === 'session' || h.payload.subcategory === 'session-chunk' ? 'session' : h.source}\0${JSON.stringify([...(h.payload.tags || [])].sort())}\0${h.content}`).digest('hex');
+    // A legacy mirrored chunk shares its UUID with the session row.
+    if (seen.has(h.id)) continue;
+    if (hybrid && byHash.has(hash)) {
+      const previous=byHash.get(hash)!;
+      const undone=d.prepare("SELECT 1 FROM memory_relations WHERE kind='duplicate_of' AND active=0 AND ((from_id=? AND to_id=?) OR (from_id=? AND to_id=?))")
+        .get(h.id,previous,previous,h.id);
+      if(!undone)continue;
+    }
+    seen.add(h.id); seen.add(hash);
+    byHash.set(hash,h.id);
+    if(h.type === 'memory') h.revision=Number((readRevision.get(h.id) as {n:number}).n || 1);
+    if(h.type==='memory' && h.payload.file_checksums && Object.keys(h.payload.file_checksums).length) {
+      const verification=d.prepare('SELECT verification,verified_revision,checked_at,stale_verdicts FROM memory_metadata WHERE memory_id=?').get(h.id);
+      if(verification?.verification==='stale')h.flags.push('source needs recheck');
+      // An edit-time judgment found this revision no longer matches its file.
+      if(verification?.stale_verdicts && judgedStale(verification.stale_verdicts,h.revision))h.flags.push('judged stale');
+      if(verification?.verified_revision!==h.revision || Date.now()-Date.parse(String(verification?.checked_at || ''))>300000 || !verification?.checked_at) {
+        d.prepare(`INSERT INTO memory_jobs(id,kind,entity_id,updated_at) VALUES(?,'verify',?,?)
+          ON CONFLICT(kind,entity_id) DO UPDATE SET status=CASE WHEN status='running' THEN status ELSE 'pending' END,attempts=0,updated_at=excluded.updated_at`)
+          .run(`verify:${h.id}`,h.id,new Date().toISOString());
+      }
+    }
+    results.push(h);
+    if (hybrid && results.length >= collect) break;
+  }
+  if (collect > limit && results.length > 1) {
+    // Rank after fusion, not instead of it. A missing or partial judgment
+    // leaves the fusion order in place; only judged hits move.
+    const judged = await judgeRelevance(o.query, results.map(h=>({id:h.id,content:excerpt(h.content,o.query,RELEVANCE_CANDIDATE_CLIP)})), o.judgeContext);
+    if (judged?.size) {
+      const ordered = orderByJudgedScore(results, judged);
+      results.splice(0, results.length, ...ordered);
+      for (const h of results) if (judged.has(h.id)) h.reasons.push('judged relevance');
+    }
+  }
+  if (collect > limit) results.length = Math.min(results.length, limit);
+  const elapsed_ms = performance.now()-start;
+  metrics.calls++; metrics.degraded += degraded.length ? 1 : 0; metrics.total_ms += elapsed_ms; metrics.last_ms = elapsed_ms;
+  recentTimes.push(elapsed_ms);if(recentTimes.length>500)recentTimes.shift();
+  mark('rank_and_metadata');
+  return { results, degraded, engine, elapsed_ms, timings };
+}
+
+/** Conservative local estimate; explicitly not a provider tokenizer. */
+export function estimateTokens(value: string) { return Math.ceil(Buffer.byteLength(value, 'utf8') / 3); }
+export function excerpt(content: string, query: string, maxChars = 700): string {
+  if (content.length <= maxChars) return content;
+  const terms = queryTerms(query);
+  const spans = content.match(/[^\n.!?]+(?:[.!?](?=\s)|\n|$)/g) || [content];
+  const ranked = spans.map((text,index) => ({text,index,score:terms.filter(t => text.toLowerCase().includes(t)).length})).sort((a,b) => b.score-a.score || a.index-b.index);
+  const chosen: typeof ranked = [];
+  let size = 0;
+  for (const span of ranked) {
+    if (span.text.length > maxChars) {
+      if (chosen.length) continue;
+      const term = terms.find(t => span.text.toLowerCase().includes(t));
+      const at = term ? span.text.toLowerCase().indexOf(term) : 0;
+      const start = Math.max(0,at-100);
+      chosen.push({...span,text:(start ? '…' : '')+span.text.slice(start,start+maxChars-2)+'…'}); break;
+    }
+    if (size+span.text.length+3 > maxChars) continue;
+    chosen.push(span); size += span.text.length+3;
+  }
+  return chosen.sort((a,b)=>a.index-b.index).map(s=>s.text.trim()).join(' … ');
+}
+export function compactRecall(result: RecallResult, query: string, budget = 1500, explain = false) {
+  const header = `SynaBun: ${result.engine} retrieval. Token counts are local estimates.${result.degraded.length ? ' '+result.degraded.join(' ') : ''}`;
+  const lines = [header];
+  for (const h of result.results) {
+    const meta = `[${h.id}] r${h.revision} ${h.type} | ${h.project} | ${h.category} | ${h.source} | ${h.created_at}${h.flags.length ? ' | '+h.flags.join(',') : ''}`;
+    const remaining = (budget-estimateTokens(lines.join('\n\n')+'\n\n'+meta+'\n'))*3;
+    if (remaining < 60) break;
+    let chars = Math.min(700,remaining);
+    let line = '';
+    do {
+      line = meta+'\n'+excerpt(h.content,query,chars)+(explain ? '\nMatched: '+h.reasons.join(', ')+'; scores: '+JSON.stringify({semantic:h.semantic_score,passage:h.passage_score,keyword_coverage:h.keyword_coverage,fusion:h.fusion_score}) : '');
+      if (estimateTokens([...lines,line].join('\n\n')) <= budget) break;
+      chars = Math.floor(chars*0.8);
+    } while (chars >= 40);
+    if (estimateTokens([...lines,line].join('\n\n')) > budget) continue;
+    lines.push(line);
+  }
+  if (lines.length === 1 && !result.results.length) lines.push('No relevant evidence found.');
+  return estimateTokens(lines.join('\n\n')) <= budget ? lines.join('\n\n') : '';
+}

package/mcp-server/src/services/memory-schema.ts

@@ -0,0 +1,280 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { createHash } from 'node:crypto';
+
+// Published migration numbers are immutable. Later additions must run even if
+// an earlier build already marked its own schema version complete.
+const migrations = [
+  { version: 2, apply: migrateV2 },
+  { version: 3, apply: migrateV3 },
+  { version: 4, apply: migrateV4 },
+  { version: 5, apply: migrateV5 },
+  { version: 6, apply: migrateV6 },
+  { version: 7, apply: migrateV7 },
+];
+
+/** Additive migrations. Triggers also cover legacy/direct SQL writers. */
+export function migrateMemory(d: DatabaseSync, dbPath: string): void {
+  d.exec('CREATE TABLE IF NOT EXISTS memory_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL, backup TEXT)');
+  const applied = new Set(d.prepare('SELECT version FROM memory_migrations').all().map(row => Number(row.version)));
+  const pending = migrations.filter(migration => !applied.has(migration.version));
+  if (!pending.length) return;
+  // VACUUM INTO makes a consistent copy including committed WAL pages. Two
+  // migrating processes may make backups; BEGIN IMMEDIATE serializes the DDL.
+  const populated = applied.size > 0 || Boolean(d.prepare('SELECT EXISTS(SELECT 1 FROM memories) OR EXISTS(SELECT 1 FROM session_chunks) AS populated').get()?.populated);
+  const backup = populated && dbPath !== ':memory:' ? `${dbPath}.pre-v${pending[0].version}-${Date.now()}-${process.pid}.bak` : null;
+  if (backup) d.prepare('VACUUM INTO ?').run(backup);
+  d.exec('BEGIN IMMEDIATE');
+  try {
+    for (const migration of pending) {
+      if (d.prepare('SELECT 1 FROM memory_migrations WHERE version = ?').get(migration.version)) continue;
+      migration.apply(d);
+      d.prepare('INSERT INTO memory_migrations(version,applied_at,backup) VALUES(?,?,?)')
+        .run(migration.version, new Date().toISOString(), backup);
+    }
+    d.exec('COMMIT');
+  } catch (error) { d.exec('ROLLBACK'); throw error; }
+}
+
+const columns = ['content','category','subcategory','project','tags','importance','source','created_at','updated_at',
+  'accessed_at','access_count','related_files','related_memory_ids','file_checksums','trashed_at','source_session_chunks'];
+const payload = (alias: string) => `json_object(${columns.map(c => `'${c}',${alias}.${c}`).join(',')})`;
+
+function migrateV2(d: DatabaseSync): void {
+    d.exec(`
+      CREATE TABLE memory_changes (seq INTEGER PRIMARY KEY AUTOINCREMENT, entity TEXT NOT NULL, id TEXT NOT NULL);
+      CREATE INDEX memory_changes_entity ON memory_changes(entity, seq);
+      CREATE INDEX idx_memory_project_lower ON memories(lower(project));
+      CREATE INDEX idx_chunk_project_lower ON session_chunks(lower(project));
+      CREATE TABLE memory_revisions (memory_id TEXT NOT NULL, revision INTEGER NOT NULL, payload TEXT NOT NULL,
+        vector BLOB NOT NULL, recorded_at TEXT NOT NULL, PRIMARY KEY(memory_id, revision));
+      CREATE TABLE memory_metadata (memory_id TEXT PRIMARY KEY, kind TEXT NOT NULL DEFAULT 'note',
+        source_ref TEXT, content_hash TEXT, idempotency_key TEXT, project TEXT, category TEXT, request_hash TEXT,
+        verification TEXT, verified_revision INTEGER, checked_at TEXT);
+      CREATE UNIQUE INDEX memory_idempotency ON memory_metadata(project, category, idempotency_key) WHERE idempotency_key IS NOT NULL;
+      CREATE TABLE memory_relations (id TEXT PRIMARY KEY, from_id TEXT NOT NULL, to_id TEXT NOT NULL,
+        kind TEXT NOT NULL CHECK(kind IN ('duplicate_of','supersedes','conflicts_with','possible_conflict','similar')),
+        automatic INTEGER NOT NULL DEFAULT 0, active INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL,
+        UNIQUE(from_id, to_id, kind));
+      CREATE INDEX memory_relations_from ON memory_relations(from_id, active);
+      CREATE INDEX memory_relations_to ON memory_relations(to_id, active);
+      CREATE TABLE memory_feedback (id TEXT PRIMARY KEY, memory_id TEXT NOT NULL, value TEXT NOT NULL, created_at TEXT NOT NULL);
+      CREATE TABLE memory_jobs (id TEXT PRIMARY KEY, kind TEXT NOT NULL, entity_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending',
+        attempts INTEGER NOT NULL DEFAULT 0, error TEXT, updated_at TEXT NOT NULL, UNIQUE(kind, entity_id));
+      CREATE TABLE memory_passages (id TEXT PRIMARY KEY, memory_id TEXT NOT NULL, content_hash TEXT NOT NULL,
+        position INTEGER NOT NULL, content TEXT NOT NULL, vector BLOB NOT NULL, model TEXT NOT NULL);
+      CREATE INDEX memory_passages_parent ON memory_passages(memory_id);
+      CREATE TABLE memory_session_sources (host TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+        content_hash TEXT NOT NULL, chunk_id TEXT NOT NULL, PRIMARY KEY(host, session_id, source_id));
+      CREATE TABLE memory_capture_files (host TEXT NOT NULL, session_id TEXT NOT NULL, file_path TEXT NOT NULL,
+        project TEXT NOT NULL, cwd TEXT, byte_offset INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'pending',
+        error TEXT, PRIMARY KEY(host,session_id));
+      CREATE VIRTUAL TABLE memory_search USING fts5(id UNINDEXED, content, category, project, tags, files, tokenize='unicode61 remove_diacritics 2');
+      CREATE VIRTUAL TABLE chunk_search USING fts5(id UNINDEXED, content, summary, project, files, tokenize='unicode61 remove_diacritics 2');
+    `);
+    // Repair the old external-content index and keep it valid for old readers.
+    d.exec(`DROP TABLE memories_fts;
+      CREATE VIRTUAL TABLE memories_fts USING fts5(content,category,project,tags,content=memories,content_rowid=rowid,tokenize='porter unicode61');
+      INSERT INTO memories_fts(memories_fts) VALUES('rebuild');
+      INSERT INTO memory_search SELECT id,content,category,project,tags,related_files FROM memories WHERE trashed_at IS NULL;
+      INSERT INTO chunk_search SELECT id,content,summary,project,files_modified FROM session_chunks;
+      INSERT INTO memory_revisions SELECT m.id,1,${payload('m')},m.vector,strftime('%Y-%m-%dT%H:%M:%fZ','now') FROM memories m;
+      INSERT INTO memory_jobs(id,kind,entity_id,updated_at) SELECT 'index:'||id,'index',id,strftime('%Y-%m-%dT%H:%M:%fZ','now') FROM memories;
+    `);
+    installMemoryTriggers(d);
+}
+
+/** Repair stores opened by early v2 builds, without replaying their migration. */
+function migrateV3(d: DatabaseSync): void {
+  const metadataColumns = new Set(d.prepare('PRAGMA table_info(memory_metadata)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({ request_hash: 'TEXT', verification: 'TEXT', verified_revision: 'INTEGER', checked_at: 'TEXT' })) {
+    if (!metadataColumns.has(name)) d.exec(`ALTER TABLE memory_metadata ADD COLUMN ${name} ${type}`);
+  }
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS memory_capture_files (host TEXT NOT NULL, session_id TEXT NOT NULL, file_path TEXT NOT NULL,
+      project TEXT NOT NULL, cwd TEXT, byte_offset INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'pending',
+      error TEXT, PRIMARY KEY(host,session_id));
+    CREATE INDEX IF NOT EXISTS idx_memory_project_lower ON memories(lower(project));
+    CREATE INDEX IF NOT EXISTS idx_chunk_project_lower ON session_chunks(lower(project));
+  `);
+  // Early retry keys predate request_hash. Recover the original request from
+  // its first revision, since later indexing may have replaced content_hash.
+  const retryRequests = d.prepare(`SELECT md.memory_id, COALESCE((
+    SELECT json_extract(r.payload,'$.content') FROM memory_revisions r
+    WHERE r.memory_id=md.memory_id ORDER BY r.revision LIMIT 1),m.content) AS content
+    FROM memory_metadata md JOIN memories m ON m.id=md.memory_id
+    WHERE md.idempotency_key IS NOT NULL AND md.request_hash IS NULL`).all();
+  const updateHash = d.prepare('UPDATE memory_metadata SET request_hash=? WHERE memory_id=?');
+  for (const row of retryRequests) {
+    updateHash.run(createHash('sha256').update(String(row.content)).digest('hex'), row.memory_id);
+  }
+  installMemoryTriggers(d);
+  // A running client may already have cached filters before these triggers
+  // existed. Invalidate that epoch once, without touching content or vectors.
+  d.exec("INSERT INTO memory_changes(entity,id) VALUES('memory_filters','schema:3')");
+}
+
+/**
+ * Room for TypeSafe's judgments beside the stored values, never replacing them.
+ * The caller's stated importance stays authoritative in `memories.importance`;
+ * `importance_judged` records what the model would have said, so agreement can
+ * be measured on real data before anything starts trusting it.
+ */
+function migrateV4(d: DatabaseSync): void {
+  const metadataColumns = new Set(d.prepare('PRAGMA table_info(memory_metadata)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({
+    importance_judged: 'REAL', importance_confidence: 'REAL',
+    kind_judged: 'TEXT', kind_confidence: 'REAL', judged_at: 'TEXT',
+  })) {
+    if (!metadataColumns.has(name)) d.exec(`ALTER TABLE memory_metadata ADD COLUMN ${name} ${type}`);
+  }
+}
+
+/**
+ * A log of every TypeSafe judgment (bounded by the writer, see
+ * typesafe-config.ts), the jobs index the judge backfill needs, room for the
+ * category check beside the other judged columns, and the duplicate_of
+ * direction repair. Nothing here touches stored content or vectors.
+ */
+function migrateV5(d: DatabaseSync): void {
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS typesafe_log (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      created_at TEXT NOT NULL,
+      surface TEXT NOT NULL,
+      origin TEXT NOT NULL DEFAULT 'live',
+      entity_id TEXT,
+      model TEXT,
+      question_count INTEGER,
+      state_preview TEXT,
+      answers TEXT,
+      latency_ms INTEGER,
+      input_tokens INTEGER,
+      output_tokens INTEGER,
+      cached INTEGER DEFAULT 0,
+      error TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_typesafe_log_created ON typesafe_log(created_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_typesafe_log_surface ON typesafe_log(surface, created_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_typesafe_log_entity ON typesafe_log(entity_id);
+    CREATE INDEX IF NOT EXISTS idx_memory_jobs_status_kind ON memory_jobs(status, kind, updated_at);
+  `);
+  const metadataColumns = new Set(d.prepare('PRAGMA table_info(memory_metadata)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({ category_judged: 'TEXT', category_confidence: 'REAL' })) {
+    if (!metadataColumns.has(name)) d.exec(`ALTER TABLE memory_metadata ADD COLUMN ${name} ${type}`);
+  }
+  repairDuplicateDirection(d);
+}
+
+/**
+ * Trash triage's expendability score beside the other judged columns, so a
+ * repeat triage reads it instead of asking again; the basis fingerprints what
+ * was judged, and a mismatch means ask again. The content_hash index turns the
+ * exact-copy lookups (triage discovery, the index job's duplicate check) from
+ * a per-row project scan into an index probe.
+ */
+function migrateV6(d: DatabaseSync): void {
+  const metadataColumns = new Set(d.prepare('PRAGMA table_info(memory_metadata)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({ expendability: 'REAL', expendability_at: 'TEXT', expendability_basis: 'TEXT' })) {
+    if (!metadataColumns.has(name)) d.exec(`ALTER TABLE memory_metadata ADD COLUMN ${name} ${type}`);
+  }
+  d.exec('CREATE INDEX IF NOT EXISTS idx_memory_metadata_content_hash ON memory_metadata(content_hash)');
+}
+
+/**
+ * Attribution for the judgment log (which session and project asked, which
+ * riders shared the request, what the caller did with the answer), and room
+ * beside each memory for the edit-time stale verdicts. Additive DDL only: old
+ * rows keep NULL sessions and their `live` origin.
+ */
+function migrateV7(d: DatabaseSync): void {
+  const logColumns = new Set(d.prepare('PRAGMA table_info(typesafe_log)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({ session_id: 'TEXT', project: 'TEXT', surfaces: 'TEXT', outcome: 'TEXT' })) {
+    if (!logColumns.has(name)) d.exec(`ALTER TABLE typesafe_log ADD COLUMN ${name} ${type}`);
+  }
+  d.exec(`
+    CREATE INDEX IF NOT EXISTS idx_typesafe_log_session ON typesafe_log(session_id, id) WHERE session_id IS NOT NULL;
+    CREATE INDEX IF NOT EXISTS idx_typesafe_log_project ON typesafe_log(project, created_at DESC) WHERE project IS NOT NULL;
+  `);
+  const metadataColumns = new Set(d.prepare('PRAGMA table_info(memory_metadata)').all().map(row => String(row.name)));
+  for (const [name, type] of Object.entries({ stale_verdicts: 'TEXT', stale_checked_at: 'TEXT' })) {
+    if (!metadataColumns.has(name)) d.exec(`ALTER TABLE memory_metadata ADD COLUMN ${name} ${type}`);
+  }
+}
+
+/**
+ * `duplicate_of` is directional: the NEWER memory points at the older one, so
+ * a pair judged from either end collides on one row. Earlier builds relied on
+ * every caller computing that direction itself; this makes stored rows match
+ * the rule `setRelation` now enforces. When both directions exist, the
+ * canonical row survives and inherits an explicit undo (active=0) from either
+ * side; lone rows pointing the wrong way are flipped in place.
+ */
+export function repairDuplicateDirection(d: DatabaseSync): { collapsed: number; flipped: number } {
+  const older = '(f.created_at<t.created_at OR (f.created_at=t.created_at AND f.id<t.id))';
+  const newer = '(f.created_at>t.created_at OR (f.created_at=t.created_at AND f.id>t.id))';
+  d.exec(`UPDATE memory_relations SET active=0 WHERE kind='duplicate_of' AND active=1 AND id IN (
+    SELECT a.id FROM memory_relations a
+    JOIN memory_relations b ON b.kind='duplicate_of' AND b.from_id=a.to_id AND b.to_id=a.from_id AND b.active=0
+    JOIN memories f ON f.id=a.from_id JOIN memories t ON t.id=a.to_id
+    WHERE a.kind='duplicate_of' AND ${newer})`);
+  const collapsed = d.prepare(`DELETE FROM memory_relations WHERE kind='duplicate_of' AND id IN (
+    SELECT a.id FROM memory_relations a
+    JOIN memory_relations b ON b.kind='duplicate_of' AND b.from_id=a.to_id AND b.to_id=a.from_id
+    JOIN memories f ON f.id=a.from_id JOIN memories t ON t.id=a.to_id
+    WHERE a.kind='duplicate_of' AND ${older})`).run().changes;
+  const flipped = d.prepare(`UPDATE memory_relations SET from_id=to_id, to_id=from_id WHERE kind='duplicate_of' AND id IN (
+    SELECT r.id FROM memory_relations r JOIN memories f ON f.id=r.from_id JOIN memories t ON t.id=r.to_id
+    WHERE r.kind='duplicate_of' AND ${older})`).run().changes;
+  return { collapsed: Number(collapsed), flipped: Number(flipped) };
+}
+
+function installMemoryTriggers(d: DatabaseSync): void {
+    for (const name of ['memory_insert','memory_update','memory_reindex','memory_vector_update','memory_delete']) {
+      d.exec(`DROP TRIGGER IF EXISTS ${name}`);
+    }
+    const revision = (alias: string) => `INSERT INTO memory_revisions SELECT ${alias}.id,
+      COALESCE((SELECT MAX(revision) FROM memory_revisions WHERE memory_id=${alias}.id),0)+1,
+      ${payload(alias)},${alias}.vector,strftime('%Y-%m-%dT%H:%M:%fZ','now');`;
+    const enqueue = `INSERT INTO memory_jobs(id,kind,entity_id,updated_at) VALUES('index:'||new.id,'index',new.id,strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+      ON CONFLICT(kind,entity_id) DO UPDATE SET status='pending',attempts=0,error=NULL,updated_at=excluded.updated_at;`;
+    const insertSearch = `INSERT INTO memory_search(id,content,category,project,tags,files)
+      SELECT new.id,new.content,new.category,new.project,new.tags,new.related_files WHERE new.trashed_at IS NULL;`;
+    d.exec(`CREATE TRIGGER memory_insert AFTER INSERT ON memories BEGIN
+        DELETE FROM memory_search WHERE id=new.id; ${insertSearch}
+        INSERT INTO memories_fts(rowid,content,category,project,tags) VALUES(new.rowid,new.content,new.category,new.project,new.tags);
+        INSERT INTO memory_changes(entity,id) VALUES('memories',new.id);
+        INSERT INTO memory_changes(entity,id) VALUES('memory_filters',new.id);
+        ${revision('new')} ${enqueue}
+      END;
+      CREATE TRIGGER memory_update AFTER UPDATE OF content,category,subcategory,project,tags,importance,source,created_at,related_files,related_memory_ids,file_checksums,trashed_at,source_session_chunks ON memories BEGIN
+        DELETE FROM memory_search WHERE id=new.id; ${insertSearch}
+        INSERT INTO memories_fts(memories_fts,rowid,content,category,project,tags) VALUES('delete',old.rowid,old.content,old.category,old.project,old.tags);
+        INSERT INTO memories_fts(rowid,content,category,project,tags) VALUES(new.rowid,new.content,new.category,new.project,new.tags);
+        INSERT INTO memory_changes(entity,id) VALUES('memory_filters',new.id);
+        ${revision('new')}
+      END;
+      CREATE TRIGGER memory_reindex AFTER UPDATE OF content,project,category,source ON memories BEGIN ${enqueue} END;
+      CREATE TRIGGER memory_vector_update AFTER UPDATE OF vector,trashed_at ON memories BEGIN
+        INSERT INTO memory_changes(entity,id) VALUES('memories',new.id);
+      END;
+      CREATE TRIGGER memory_delete AFTER DELETE ON memories BEGIN
+        DELETE FROM memory_search WHERE id=old.id;
+        INSERT INTO memories_fts(memories_fts,rowid,content,category,project,tags) VALUES('delete',old.rowid,old.content,old.category,old.project,old.tags);
+        INSERT INTO memory_changes(entity,id) VALUES('memories',old.id);
+        INSERT INTO memory_changes(entity,id) VALUES('memory_filters',old.id);
+        DELETE FROM memory_passages WHERE memory_id=old.id;
+        DELETE FROM memory_jobs WHERE entity_id=old.id;
+      END;
+    `);
+    for (const table of ['session_chunks', 'memory_passages']) {
+      for (const event of ['INSERT','UPDATE','DELETE']) {
+        const alias = event === 'DELETE' ? 'old' : 'new';
+        d.exec(`DROP TRIGGER IF EXISTS ${table}_${event.toLowerCase()};
+          CREATE TRIGGER ${table}_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN
+          INSERT INTO memory_changes(entity,id) VALUES('${table}',${alias}.id);
+          ${table === 'session_chunks' ? `DELETE FROM chunk_search WHERE id=${alias}.id;
+            ${event !== 'DELETE' ? 'INSERT INTO chunk_search(id,content,summary,project,files) VALUES(new.id,new.content,new.summary,new.project,new.files_modified);' : ''}` : ''}
+        END;`);
+      }
+    }
+}