renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -10,6 +10,7 @@ exports.unterminatedFinalEventStart = unterminatedFinalEventStart;
10
10
  exports.parseLedgerEvents = parseLedgerEvents;
11
11
  exports.replayLedger = replayLedger;
12
12
  exports.remember = remember;
13
+ exports.eraseRetiredFact = eraseRetiredFact;
13
14
  exports.factEventId = factEventId;
14
15
  exports.retireFact = retireFact;
15
16
  exports.restoreFact = restoreFact;
@@ -32,8 +33,11 @@ const invisibleText_1 = require("./invisibleText");
32
33
  const sourceName_1 = require("./sourceName");
33
34
  const writerConflicts_1 = require("./writerConflicts");
34
35
  const durableAppend_1 = require("./durableAppend");
36
+ const writer_1 = require("./writer");
35
37
  const secretGuard_1 = require("../extraction/secretGuard");
36
38
  const toolRefusal_1 = require("../toolRefusal");
39
+ const log_1 = require("../mvp/log");
40
+ const log = (0, log_1.createLogger)('engine');
37
41
  /** Reserved synthetic note path owning lifecycle entities (never a real vault file). */
38
42
  exports.LIFECYCLE_NOTE_PATH = '_mcp_';
39
43
  /** Default relative ledger path inside the vault. */
@@ -44,7 +48,7 @@ const EVENT_HEADER_RE = /^## (mem_[A-Za-z0-9]+) · (.+?)[ \t]*$/i;
44
48
  // `.` otherwise refuses U+2028/U+2029 (and a lone \r), which JavaScript calls line terminators and this
45
49
  // ledger does not: one such character in a value used to make the whole ledger unreadable at startup.
46
50
  const FIELD_RE = /^- ([A-Za-z][\w-]*): (.*)$/s;
47
- const KNOWN_FIELDS = new Set(['kind', 'subject', 'predicate', 'object', 'closed_event', 'source', 'scope', 'statement', 'writer', 'target']);
51
+ const KNOWN_FIELDS = new Set(['kind', 'subject', 'predicate', 'object', 'closed_event', 'source', 'scope', 'statement', 'writer', 'target', 'original_digest', 'closes', 'closed_at']);
48
52
  const REQUIRED_FIELDS = ['subject', 'predicate', 'object'];
49
53
  /**
50
54
  * retire takes one fact out of recall and restore brings it back (retireFact, restoreFact). Each names the fact's own
@@ -53,7 +57,8 @@ const REQUIRED_FIELDS = ['subject', 'predicate', 'object'];
53
57
  * is written.
54
58
  */
55
59
  const RETIREMENT_KINDS = new Set(['retire', 'restore']);
56
- const KNOWN_KINDS = new Set(['store', 'supersede', ...RETIREMENT_KINDS]);
60
+ const ERASE_KIND = 'erase';
61
+ const KNOWN_KINDS = new Set(['store', 'supersede', ...RETIREMENT_KINDS, ERASE_KIND]);
57
62
  const CONTROL_CHARS = /[\u0000-\u001F\u007F]/;
58
63
  exports.FIELD_LIMITS = { subject: 512, predicate: 256, object: 8192, source: 256, scope: 80, statement: limits_1.MAX_STATEMENT_LENGTH, writer: 256 };
59
64
  /**
@@ -243,6 +248,24 @@ function validateLedgerEvent(ev) {
243
248
  }
244
249
  if (!KNOWN_KINDS.has(ev.kind))
245
250
  throw new Error(`ledger corruption in event ${ev.id}: unknown kind "${ev.kind}"`);
251
+ if (ev.kind === ERASE_KIND) {
252
+ if (!/^[0-9a-f]{64}$/.test(ev.originalDigest ?? ''))
253
+ throw new Error(`ledger corruption in event ${ev.id}: an erase tombstone needs the original event digest`);
254
+ if (ev.subject || ev.predicate || ev.object || ev.statement !== undefined || ev.writer !== undefined || ev.target !== undefined || ev.closedEvent !== null || ev.source !== 'erased') {
255
+ throw new Error(`ledger corruption in event ${ev.id}: an erase tombstone carries no memory content`);
256
+ }
257
+ // The closing link is ids and a time, both checked, so it cannot carry text either.
258
+ if ((ev.closes === undefined) !== (ev.closedAt === undefined))
259
+ throw new Error(`ledger corruption in event ${ev.id}: an erase tombstone's closes and closed_at go together`);
260
+ if (ev.closes !== undefined) {
261
+ if (ev.closes.length === 0 || ev.closes.some(id => !/^MEM_[A-Z0-9]+$/.test(id)))
262
+ throw new Error(`ledger corruption in event ${ev.id}: an erase tombstone closes event ids only`);
263
+ if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(ev.closedAt) || new Date(ev.closedAt).toISOString() !== ev.closedAt) {
264
+ throw new Error(`ledger corruption in event ${ev.id}: an erase tombstone's closed_at is not an exact UTC ISO time`);
265
+ }
266
+ }
267
+ return;
268
+ }
246
269
  const retirement = RETIREMENT_KINDS.has(ev.kind);
247
270
  validateField('subject', ev.subject, true);
248
271
  validateField('predicate', ev.predicate, true);
@@ -283,6 +306,8 @@ function validateLedgerEvent(ev) {
283
306
  }
284
307
  }
285
308
  function eventDigest(ev) {
309
+ if (ev.kind === ERASE_KIND)
310
+ return ev.originalDigest;
286
311
  const parts = [
287
312
  ev.id, ev.at, ev.kind, ev.subject, ev.predicate, ev.object, ev.closedEvent ?? '', ev.source ?? 'ledger',
288
313
  ];
@@ -357,31 +382,14 @@ function resolveLedgerPath(vaultPath, ledgerRelPath) {
357
382
  }
358
383
  async function appendLedgerEvent(vaultPath, ledgerRelPath, ev) {
359
384
  validateLedgerEvent(ev);
385
+ // An erasure is a rewrite of the whole ledger (eraseRetiredFact), never an append.
386
+ if (ev.kind === ERASE_KIND)
387
+ throw new Error(`ledger event ${ev.id}: an erase tombstone is written only by erasing a retired fact`);
360
388
  const resolved = resolveLedgerPath(vaultPath, ledgerRelPath);
361
- const lines = [
362
- `\n## ${ev.id} · ${ev.at}`,
363
- `- kind: ${ev.kind}`,
364
- `- subject: ${ev.subject}`,
365
- `- predicate: ${ev.predicate}`,
366
- ];
367
- if (ev.target !== undefined)
368
- lines.push(`- target: ${ev.target}`);
369
- else
370
- lines.push(`- object: ${ev.object}`);
371
- if (ev.closedEvent)
372
- lines.push(`- closed_event: ${ev.closedEvent}`);
373
- const scope = (0, scope_1.normalizeScope)(ev.scope);
374
- if (scope !== 'global')
375
- lines.push(`- scope: ${scope}`);
376
- if (ev.statement !== undefined)
377
- lines.push(`- statement: ${encodeStatement(ev.statement)}`);
378
- if (ev.writer !== undefined)
379
- lines.push(`- writer: ${ev.writer}`);
380
- lines.push(`- source: ${ev.source}`, ``);
381
389
  // Confined and durable in one append (confinedWrite.ts): a link, a hard link or a FIFO planted in the vault is refused,
382
390
  // never followed, and a failed write is cut back, so the ledger never keeps a fragment of an event whose save the caller
383
391
  // was told failed. A new ledger starts with its header.
384
- await (0, confinedWrite_1.appendConfined)(vaultPath, path_1.default.relative(path_1.default.resolve(vaultPath), resolved), lines.join('\n'), LEDGER_HEADER);
392
+ await (0, confinedWrite_1.appendConfined)(vaultPath, path_1.default.relative(path_1.default.resolve(vaultPath), resolved), ledgerEventBlock(ev), LEDGER_HEADER);
385
393
  return resolved;
386
394
  }
387
395
  const LEDGER_HEADER = `---\ntype: memory-ledger\ntitle: Memory Events Ledger\n---\n\n# Memory Events Ledger\n\nAppend-only durable record of memory events. Replayed on rebuild.\n`;
@@ -475,7 +483,13 @@ async function moveTornTailAside(vaultPath, relFile, ledger, bytes, offset) {
475
483
  }
476
484
  await ledger.truncate(offset);
477
485
  await ledger.sync();
478
- console.error(`[memory] ledger ${path_1.default.resolve(vaultPath, relFile)}: its final event was cut short (a crash or a full disk); moved its ${tornBytes.length} byte(s) to ${tornPath} and kept every complete event. If that file holds a hand-written event that was never replayed, restore it by adding its \`- source:\` line and a final newline (if it has none), appending its bytes to the ledger, and restarting; a block still without that line is set aside again.`);
486
+ log.error('ledger_tail_set_aside', {
487
+ ledger: relFile,
488
+ bytes: tornBytes.length,
489
+ movedTo: path_1.default.relative(vaultPath, tornPath),
490
+ detail: 'the final event was cut short (a crash or a full disk); every complete event was kept',
491
+ remedy: 'a hand-written event never replayed: add its `- source:` line and a final newline (if it has none), append its bytes to the ledger and restart; a block still without that line is set aside again',
492
+ });
479
493
  }
480
494
  /**
481
495
  * The ledger's text for replay, and whether its unterminated final event (if any) is one the index already holds. An
@@ -500,6 +514,23 @@ async function readLedgerForReplay(vaultPath, filePath, isProjected, setTornTail
500
514
  return { content: again.subarray(0, now.offset).toString('utf8'), finalEventKnown: false };
501
515
  }));
502
516
  }
517
+ /** Remove private rewrite files left by a process that died before replacing the ledger. */
518
+ async function removeStaleEraseTemps(filePath) {
519
+ const dir = path_1.default.dirname(filePath);
520
+ const prefix = `${path_1.default.basename(filePath)}.erase-`;
521
+ let names;
522
+ try {
523
+ names = await promises_1.default.readdir(dir);
524
+ }
525
+ catch (err) {
526
+ if (err.code === 'ENOENT')
527
+ return;
528
+ throw err;
529
+ }
530
+ await Promise.all(names
531
+ .filter(name => name.startsWith(prefix) && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.tmp$/i.test(name.slice(prefix.length)))
532
+ .map(name => promises_1.default.rm(path_1.default.join(dir, name), { force: true })));
533
+ }
503
534
  /**
504
535
  * Strict line-oriented parser for the ledger format. Returns the events in
505
536
  * append order. Throws (never silently skips) on event-shaped corruption:
@@ -555,6 +586,8 @@ function parseLedgerEvents(content, options = {}) {
555
586
  }
556
587
  const kind = fields.kind ?? 'store';
557
588
  for (const req of REQUIRED_FIELDS) {
589
+ if (kind === ERASE_KIND)
590
+ continue;
558
591
  if (req === 'object' && RETIREMENT_KINDS.has(kind))
559
592
  continue; // a retirement carries no value
560
593
  if (fields[req] === undefined || fields[req] === '') {
@@ -565,6 +598,9 @@ function parseLedgerEvents(content, options = {}) {
565
598
  id, at,
566
599
  ...(fields.statement !== undefined ? { statement: decodeStatement(fields.statement, id) } : {}),
567
600
  ...(fields.target !== undefined ? { target: fields.target } : {}),
601
+ ...(fields.original_digest !== undefined ? { originalDigest: fields.original_digest } : {}),
602
+ ...(fields.closes !== undefined ? { closes: fields.closes.split(',').map(part => part.trim().toUpperCase()) } : {}),
603
+ ...(fields.closed_at !== undefined ? { closedAt: fields.closed_at } : {}),
568
604
  kind,
569
605
  subject: fields.subject,
570
606
  predicate: fields.predicate,
@@ -853,35 +889,11 @@ function requireLedgerPosition(positions, eventId) {
853
889
  }
854
890
  return pos;
855
891
  }
856
- /**
857
- * Canonical stream reconciliation (K12): recompute the current/valid_to chain
858
- * for every event-backed lifecycle fact of (lifecycle entity, predicate) from
859
- * the CANONICAL ledger positions, exactly mirroring what a fresh in-order
860
- * replay produces — the event that appears latest in the ledger is the
861
- * current fact; each earlier event's row is closed at its canonical
862
- * successor's valid_from (never before its own valid_from, so no negative
863
- * intervals). Stale winner flags left behind by earlier candidate reconciliations
864
- * (including migration 6's interim duplicate repair, which cannot read the
865
- * ledger) are repaired here, before readiness. Non-event-backed open rows are
866
- * non-canonical (base vault_store_fact bypassed the ledger): they are closed
867
- * at the start of the canonical record and KEPT in history — never dropped.
868
- * Event-backed rows whose event is absent from the ledger are an
869
- * index-vs-ledger contradiction and fail explicitly. Unrelated (non '_mcp_')
870
- * facts are not touched. Idempotent: re-running produces no further writes.
871
- *
872
- * The stream is the scope's slug (streamCurrentFacts): the rows of every
873
- * spelling are replayed together, in ledger order, as applyEvent applies them.
874
- * Each event closes the current row of its own spelling and the row its
875
- * closed_event names under another spelling (currentFactClosedAcrossSpellings;
876
- * `closedBy` holds each event's closed_event). Without such a link the
877
- * spellings never touch, so each spelling's chain is exactly the per-spelling
878
- * chain this function computed before scopes were one slug.
879
- */
880
- function reconcileStreamCurrent(db, lifecycleId, predicate, positions, scope = 'global', closedBy = new Map()) {
881
- (0, statements_1.transactionOnce)(db, reconcileStreamCurrentBody)(lifecycleId, predicate, positions, scope, closedBy);
892
+ function reconcileStreamCurrent(db, lifecycleId, predicate, positions, scope = 'global', closedBy = new Map(), closers = new Map()) {
893
+ (0, statements_1.transactionOnce)(db, reconcileStreamCurrentBody)(lifecycleId, predicate, positions, scope, closedBy, closers);
882
894
  }
883
895
  /** The body of reconcileStreamCurrent's transaction: a module-level function so replay wraps it once per connection (transactionOnce). */
884
- function reconcileStreamCurrentBody(db, lifecycleId, predicate, positions, scope, closedBy) {
896
+ function reconcileStreamCurrentBody(db, lifecycleId, predicate, positions, scope, closedBy, closers) {
885
897
  const inStream = (0, scope_1.savedScopeSqlPredicate)(scope, 'scope');
886
898
  // A retired fact keeps its place in the stream (retireFact): it is hidden, not undone, so the value it closed stays
887
899
  // closed and the value saved after it closes it, exactly as if it had never been retired.
@@ -906,7 +918,25 @@ function reconcileStreamCurrentBody(db, lifecycleId, predicate, positions, scope
906
918
  // a state a fresh replay could not produce. (Equal timestamps are legal.)
907
919
  const current = new Map();
908
920
  const closeAt = new Map();
909
- for (const row of canonical) {
921
+ // An erased fact has no row any more, but at its place in the ledger it still closes what it closed, at its own time, as a
922
+ // fresh replay of its tombstone does (applyEvent); it never becomes current itself.
923
+ const inChain = new Set(canonical.map(row => row.event_id));
924
+ const links = [...closers]
925
+ .filter(([id, link]) => positions.has(id) && link.closes.some(closed => inChain.has(closed)))
926
+ .map(([id, link]) => ({ pos: positions.get(id), ...link }));
927
+ const steps = [...canonical.map(row => ({ pos: row.pos, row })), ...links.map(link => ({ pos: link.pos, link }))]
928
+ .sort((x, y) => x.pos - y.pos);
929
+ for (const step of steps) {
930
+ if ('link' in step) {
931
+ for (const [spelling, open] of [...current]) {
932
+ if (!step.link.closes.includes(open.event_id))
933
+ continue;
934
+ closeAt.set(open.id, maxInstant(step.link.closedAt, open.valid_from));
935
+ current.delete(spelling);
936
+ }
937
+ continue;
938
+ }
939
+ const row = step.row;
910
940
  const named = closedBy.get(row.event_id);
911
941
  const across = named === undefined ? undefined
912
942
  : [...current.values()].find(r => r.scope !== row.scope && r.event_id === named);
@@ -951,14 +981,14 @@ function reconcileStreamCurrentBody(db, lifecycleId, predicate, positions, scope
951
981
  * The transaction applyEvent runs for an event that is already projected: re-home a fact still attached to a disposable entity,
952
982
  * then reconcile its stream. A module-level function so replay wraps it once per connection (transactionOnce).
953
983
  */
954
- function reconcileProjectedEvent(db, ev, factId, rehome, positions, lifecycleId, closedBy) {
984
+ function reconcileProjectedEvent(db, ev, factId, rehome, positions, lifecycleId, closedBy, closers) {
955
985
  if (rehome) {
956
986
  rehomeLifecycleFact(db, ev, factId, positions, lifecycleId);
957
987
  }
958
988
  // Do not preserve stale winner/valid_to flags merely because the event
959
989
  // is already projected: recompute the chain from canonical ledger order
960
990
  // so the upgraded cache is equivalent to a fresh replay of the ledger.
961
- reconcileStreamCurrent(db, lifecycleId, ev.predicate, positions, (0, scope_1.normalizeScope)(ev.scope), closedBy);
991
+ reconcileStreamCurrent(db, lifecycleId, ev.predicate, positions, (0, scope_1.normalizeScope)(ev.scope), closedBy, closers);
962
992
  }
963
993
  /**
964
994
  * Apply one memory event to SQLite inside a single transaction:
@@ -966,13 +996,72 @@ function reconcileProjectedEvent(db, ev, factId, rehome, positions, lifecycleId,
966
996
  * Idempotent by event id + digest; a reused id with a conflicting payload is
967
997
  * explicit corruption; a failed insert can never leave a partial supersession.
968
998
  */
969
- function applyEvent(db, ev, positions, index, closedBy) {
999
+ function applyEvent(db, ev, positions, index, closedBy, closers) {
970
1000
  validateLedgerEvent(ev);
971
1001
  const digest = eventDigest(ev);
972
1002
  const seen = (0, factTerms_1.prepareOnce)(db, 'SELECT digest FROM applied_ledger_events WHERE event_id = ?').get(ev.id);
973
1003
  if (seen && seen.digest !== digest) {
974
1004
  throw new Error(`ledger event id ${ev.id} was reused with a conflicting payload; refusing to apply (corruption)`);
975
1005
  }
1006
+ if (ev.kind === ERASE_KIND) {
1007
+ db.transaction(() => {
1008
+ const entityIds = db.prepare('SELECT subject_entity_id FROM facts WHERE event_id = ? UNION SELECT subject_entity_id FROM retired_facts WHERE event_id = ?')
1009
+ .pluck().all(ev.id, ev.id);
1010
+ const ids = db.prepare('SELECT id FROM facts WHERE event_id = ? UNION SELECT id FROM retired_facts WHERE event_id = ?')
1011
+ .all(ev.id, ev.id);
1012
+ // The erased value can also live on in the writer conflicts (writerConflicts.ts): a later fact that replaced it keeps
1013
+ // it as an open displacement, carried forward by its writer's next saves. Drop those and the erased fact's own row.
1014
+ const gone = db.prepare('SELECT object_text, valid_from, author FROM facts WHERE event_id = ? UNION SELECT object_text, valid_from, author FROM retired_facts WHERE event_id = ?')
1015
+ .get(ev.id, ev.id);
1016
+ if (gone) {
1017
+ const goneWriter = (0, writerConflicts_1.writerOf)(db, ev.id, gone.author).writer;
1018
+ const rows = db.prepare(`SELECT event_id, displaced FROM fact_writers WHERE displaced <> '[]'`).all();
1019
+ for (const row of rows) {
1020
+ const displaced = JSON.parse(row.displaced);
1021
+ const kept = displaced.filter(d => !(d.object === gone.object_text && d.validFrom === gone.valid_from && d.writer === goneWriter));
1022
+ if (kept.length !== displaced.length)
1023
+ db.prepare('UPDATE fact_writers SET displaced = ? WHERE event_id = ?').run(JSON.stringify(kept), row.event_id);
1024
+ }
1025
+ }
1026
+ db.prepare('DELETE FROM fact_writers WHERE event_id = ?').run(ev.id);
1027
+ for (const row of ids)
1028
+ db.prepare('DELETE FROM fact_terms WHERE fact_id = ?').run(row.id);
1029
+ db.prepare('DELETE FROM facts WHERE event_id = ?').run(ev.id);
1030
+ db.prepare('DELETE FROM retired_facts WHERE event_id = ?').run(ev.id);
1031
+ db.prepare('DELETE FROM fact_retirements WHERE target = ?').run(ev.id);
1032
+ db.prepare(`DELETE FROM write_events WHERE note_path = '_memory_ledger_' AND action IN ('retire', 'restore')
1033
+ AND json_valid(summary) AND json_extract(summary, '$.id') = ?`).run(`fact:${ev.id}`);
1034
+ for (const entityId of entityIds) {
1035
+ const used = db.prepare('SELECT 1 FROM facts WHERE subject_entity_id = ? UNION SELECT 1 FROM retired_facts WHERE subject_entity_id = ? LIMIT 1').get(entityId, entityId);
1036
+ if (!used)
1037
+ db.prepare(`DELETE FROM entities WHERE id = ? AND source_note = '_mcp_'`).run(entityId);
1038
+ }
1039
+ // What the erased fact closed stays closed: on a rebuild nothing else in the ledger closes it any more. A fact still open
1040
+ // here (a fresh replay) is closed at the erased fact's own time; one already closed, or gone, is left as it is.
1041
+ if (ev.closes !== undefined) {
1042
+ for (const closed of ev.closes) {
1043
+ for (const table of ['facts', 'retired_facts']) {
1044
+ db.prepare(`UPDATE ${table} SET valid_to = ?, updated_at = ? WHERE event_id = ? AND valid_to IS NULL`).run(ev.closedAt, ev.closedAt, closed);
1045
+ }
1046
+ }
1047
+ }
1048
+ // Store the old digest under the old id: verification still detects a changed pre-erasure event, while no payload remains.
1049
+ db.prepare('INSERT OR REPLACE INTO applied_ledger_events (event_id, digest, applied_at) VALUES (?, ?, ?)')
1050
+ .run(ev.id, ev.originalDigest, ev.at);
1051
+ // A tombstone without a closing link (written before there was one) keeps a link already recorded here.
1052
+ db.prepare(`INSERT INTO erased_facts (event_id, original_digest, erased_at, closes, closed_at) VALUES (?, ?, ?, ?, ?)
1053
+ ON CONFLICT(event_id) DO UPDATE SET original_digest = excluded.original_digest, erased_at = excluded.erased_at,
1054
+ closes = COALESCE(excluded.closes, erased_facts.closes), closed_at = COALESCE(excluded.closed_at, erased_facts.closed_at)`)
1055
+ .run(ev.id, ev.originalDigest, ev.at, ev.closes !== undefined ? ev.closes.join(',') : null, ev.closedAt ?? null);
1056
+ })();
1057
+ return { applied: seen === undefined, factId: null };
1058
+ }
1059
+ // An erased fact's own event, or a retirement of it, that turns up again (a stale copy, a restored fragment): its digest
1060
+ // is the one the tombstone kept, so the check above passes, and the missing-fact repair below would project it again.
1061
+ // Erasure is final: skip it, whichever came first in this ledger.
1062
+ if ((0, factTerms_1.prepareOnce)(db, 'SELECT 1 FROM erased_facts WHERE event_id IN (?, ?)').get(ev.id, ev.target?.toUpperCase() ?? ev.id)) {
1063
+ return { applied: false, factId: null };
1064
+ }
976
1065
  if (RETIREMENT_KINDS.has(ev.kind)) {
977
1066
  applyRetirement(db, ev, digest, seen !== undefined);
978
1067
  return { applied: seen === undefined, factId: null };
@@ -1000,7 +1089,7 @@ function applyEvent(db, ev, positions, index, closedBy) {
1000
1089
  // transactions become savepoints): a rejected ledger (out-of-order chain,
1001
1090
  // absent-from-ledger contradiction) rolls the reconciliation back and
1002
1091
  // leaves the cache exactly as it was held — never partially reconciled.
1003
- (0, statements_1.transactionOnce)(db, reconcileProjectedEvent)(ev, factRow.id, liveRow !== undefined && owner?.source_note !== exports.LIFECYCLE_NOTE_PATH, positions, lifecycleId, closedBy);
1092
+ (0, statements_1.transactionOnce)(db, reconcileProjectedEvent)(ev, factRow.id, liveRow !== undefined && owner?.source_note !== exports.LIFECYCLE_NOTE_PATH, positions, lifecycleId, closedBy, closers);
1004
1093
  // If it predates the digest table (migration 6), backfill the digest so
1005
1094
  // future replays can tell identical dedup from conflicts (the legacy
1006
1095
  // payload itself can no longer be verified).
@@ -1222,12 +1311,15 @@ function settleRetirements(db, events) {
1222
1311
  /** Replay the ledger into facts. Idempotent by event id + digest. Throws on corruption. */
1223
1312
  async function replayLedger(db, vaultPath, ledgerRelPath, options = {}) {
1224
1313
  const filePath = resolveLedgerPath(vaultPath, ledgerRelPath); // shared confinement, before any read
1314
+ if (options.setTornTailAside === true)
1315
+ await removeStaleEraseTemps(filePath);
1225
1316
  let content;
1226
1317
  let finalEventKnown = false;
1227
1318
  try {
1228
1319
  ({ content, finalEventKnown } = await readLedgerForReplay(vaultPath, filePath, eventId => (0, factTerms_1.prepareOnce)(db, 'SELECT 1 FROM applied_ledger_events WHERE event_id = ?').get(eventId) !== undefined, options.setTornTailAside === true));
1229
1320
  }
1230
- catch (err) {
1321
+ catch (caught) {
1322
+ const err = caught;
1231
1323
  if (err?.code === 'ENOENT') {
1232
1324
  // Only an absent ledger means "known-new empty vault: nothing recorded yet".
1233
1325
  // If the index already projects canonical events, the authoritative ledger
@@ -1261,6 +1353,9 @@ async function replayLedger(db, vaultPath, ledgerRelPath, options = {}) {
1261
1353
  const positions = new Map();
1262
1354
  // Each event's closed_event: reconciling a stream replays the facts closed across spellings (reconcileStreamCurrent).
1263
1355
  const closedBy = new Map();
1356
+ // What each erased fact closed (reconcileStreamCurrent): its tombstone's link, or, when a lost rename left its content event
1357
+ // in the ledger, the link the index kept for it.
1358
+ const closers = new Map();
1264
1359
  for (let i = 0; i < events.length; i++) {
1265
1360
  if (positions.has(events[i].id))
1266
1361
  continue;
@@ -1268,15 +1363,21 @@ async function replayLedger(db, vaultPath, ledgerRelPath, options = {}) {
1268
1363
  const closed = events[i].closedEvent;
1269
1364
  if (closed)
1270
1365
  closedBy.set(events[i].id, closed.toUpperCase());
1366
+ if (events[i].kind === ERASE_KIND && events[i].closes !== undefined)
1367
+ closers.set(events[i].id, { closes: events[i].closes, closedAt: events[i].closedAt });
1271
1368
  // A retirement names a fact event written before it.
1272
1369
  const target = events[i].target?.toUpperCase();
1273
1370
  if (target !== undefined) {
1274
1371
  const at = positions.get(target);
1275
- if (at === undefined || RETIREMENT_KINDS.has(events[at].kind)) {
1372
+ if (at === undefined || RETIREMENT_KINDS.has(events[at].kind) || events[at].kind === ERASE_KIND) {
1276
1373
  throw new Error(`ledger corruption in event ${events[i].id}: its target ${target} is no fact event before it in the ledger`);
1277
1374
  }
1278
1375
  }
1279
1376
  }
1377
+ for (const kept of (0, factTerms_1.prepareOnce)(db, 'SELECT event_id, closes, closed_at FROM erased_facts WHERE closes IS NOT NULL AND closed_at IS NOT NULL').all()) {
1378
+ if (positions.has(kept.event_id) && !closers.has(kept.event_id))
1379
+ closers.set(kept.event_id, { closes: kept.closes.split(','), closedAt: kept.closed_at });
1380
+ }
1280
1381
  // GLOBAL preflight (K13): every event-backed canonical projection in the
1281
1382
  // index must be explainable by the COMPLETE parsed ledger — not just the
1282
1383
  // streams this replay happens to touch. The stream-scoped refusal (re-home
@@ -1299,7 +1400,7 @@ async function replayLedger(db, vaultPath, ledgerRelPath, options = {}) {
1299
1400
  }
1300
1401
  const index = replayEntityIndex(db);
1301
1402
  for (const ev of events) {
1302
- if (applyEvent(db, ev, positions, index, closedBy).applied)
1403
+ if (applyEvent(db, ev, positions, index, closedBy, closers).applied)
1303
1404
  applied++;
1304
1405
  }
1305
1406
  settleRetirements(db, events);
@@ -1524,6 +1625,153 @@ async function rememberCritical(db, vaultPath, ledgerRelPath, sent) {
1524
1625
  created: { factId, object: args.object, validFrom: at }, ledger,
1525
1626
  };
1526
1627
  }
1628
+ /** One event as the ledger stores it: what an append writes, and what an erasure's rewrite writes for every event it keeps. */
1629
+ function ledgerEventBlock(ev) {
1630
+ const lines = [`\n## ${ev.id} · ${ev.at}`, `- kind: ${ev.kind}`];
1631
+ if (ev.kind === ERASE_KIND) {
1632
+ const closing = ev.closes !== undefined ? [`- closes: ${ev.closes.join(', ')}`, `- closed_at: ${ev.closedAt}`] : [];
1633
+ return [...lines, `- original_digest: ${ev.originalDigest}`, ...closing, '- source: erased', ''].join('\n');
1634
+ }
1635
+ lines.push(`- subject: ${ev.subject}`, `- predicate: ${ev.predicate}`);
1636
+ if (ev.target !== undefined)
1637
+ lines.push(`- target: ${ev.target}`);
1638
+ else
1639
+ lines.push(`- object: ${ev.object}`);
1640
+ if (ev.closedEvent)
1641
+ lines.push(`- closed_event: ${ev.closedEvent}`);
1642
+ const scope = (0, scope_1.normalizeScope)(ev.scope);
1643
+ if (scope !== 'global')
1644
+ lines.push(`- scope: ${scope}`);
1645
+ if (ev.statement !== undefined)
1646
+ lines.push(`- statement: ${encodeStatement(ev.statement)}`);
1647
+ if (ev.writer !== undefined)
1648
+ lines.push(`- writer: ${ev.writer}`);
1649
+ lines.push(`- source: ${ev.source}`, '');
1650
+ return lines.join('\n');
1651
+ }
1652
+ /** Atomically replace a retired fact and its retire/restore events with one content-free, digest-bearing tombstone. */
1653
+ async function eraseRetiredFact(db, vaultPath, ledgerRelPath, id, vectorIndex) {
1654
+ return serializeLifecycleWrite(async () => {
1655
+ const target = factEventId(id);
1656
+ if (target === null)
1657
+ throw new LifecycleInputError('id', 'a live memory is retired first; pass a retired fact id', 'invalid_argument');
1658
+ const row = db.prepare(`SELECT r.subject_entity_id FROM retired_facts r WHERE r.event_id = ?`).get(target);
1659
+ const erased = db.prepare('SELECT original_digest, erased_at, closes FROM erased_facts WHERE event_id = ?').get(target);
1660
+ if (!row && !erased) {
1661
+ if (db.prepare('SELECT 1 FROM facts WHERE event_id = ?').get(target))
1662
+ throw new LifecycleInputError('id', 'that fact is still live; retire it before deleting it permanently', 'invalid_argument');
1663
+ throw new LifecycleInputError('id', 'no retired fact has that id', 'memory_not_found');
1664
+ }
1665
+ const file = resolveLedgerPath(vaultPath, ledgerRelPath);
1666
+ const rel = path_1.default.relative(path_1.default.resolve(vaultPath), file);
1667
+ // The append lock keeps every ledger writer out; the per-file lock the indexer holds for a whole pass (indexFile) keeps
1668
+ // out an index pass that read the ledger before this rewrite and would otherwise commit its chunks after the derived
1669
+ // note is dropped below, putting the erased text back into search. Always in this order: the indexer takes no append lock.
1670
+ return (0, durableAppend_1.withAppendLock)(file, () => (0, writer_1.withDailyFileLock)(file, async () => {
1671
+ const events = parseLedgerEvents(await promises_1.default.readFile(file, 'utf8'));
1672
+ const original = events.find(ev => ev.id === target);
1673
+ if (original?.kind === ERASE_KIND || (!original && erased)) {
1674
+ // The ledger already holds the tombstone (a retry, or a crash after the rename): settle the projection and the audit.
1675
+ const at = original?.at ?? erased.erased_at;
1676
+ const ledgerChunkIds = ledgerChunks(db, file, rel);
1677
+ db.transaction(() => {
1678
+ db.prepare('DELETE FROM notes WHERE path IN (?, ?)').run(file, rel);
1679
+ if (original)
1680
+ applyEvent(db, original);
1681
+ const audit = db.prepare(`SELECT 1 FROM write_events WHERE note_path = '_retired_' AND action = 'delete' AND summary = 'deleted permanently: 1 fact' AND created_at = ?`).get(at);
1682
+ if (!audit)
1683
+ db.prepare(`INSERT INTO write_events (note_path, action, summary, created_at) VALUES ('_retired_', 'delete', 'deleted permanently: 1 fact', ?)`).run(at);
1684
+ })();
1685
+ for (const chunkId of ledgerChunkIds)
1686
+ vectorIndex?.delete(chunkId);
1687
+ return { ok: true, action: 'deleted', ids: [`fact:${target}`], freedBytes: 0, at };
1688
+ }
1689
+ if (!original || RETIREMENT_KINDS.has(original.kind))
1690
+ throw new Error(`ledger corruption: retired fact ${target} has no content-bearing event`);
1691
+ // The index says the fact was erased, yet the ledger still holds it: the rename was lost (a power loss after the index
1692
+ // committed, before the directory reached the disk). Redo the rewrite with the erasure's own digest and time.
1693
+ if (erased && eventDigest(original) !== erased.original_digest) {
1694
+ throw new Error(`ledger corruption: erased fact ${target} is back in the ledger with a different payload`);
1695
+ }
1696
+ const at = erased?.erased_at ?? new Date().toISOString();
1697
+ const subjectEntityId = row?.subject_entity_id ?? resolveLifecycleEntity(db, original.subject, original.at, false);
1698
+ // A redo after a lost rename reuses the link the first erasure recorded: a restart in between replays the old ledger
1699
+ // without the erased fact and may have reopened what it closed, so the facts tables alone no longer show it.
1700
+ const closes = [...new Set([...(erased?.closes ? erased.closes.split(',') : []), ...factsClosedBy(db, original, subjectEntityId)])].sort();
1701
+ const tombstone = {
1702
+ id: target, at, kind: ERASE_KIND, subject: '', predicate: '', object: '', closedEvent: null, source: 'erased',
1703
+ originalDigest: eventDigest(original),
1704
+ ...(closes.length > 0 ? { closes, closedAt: original.at } : {}),
1705
+ };
1706
+ const kept = events.filter(ev => !(RETIREMENT_KINDS.has(ev.kind) && ev.target?.toUpperCase() === target)).map(ev => ev.id === target ? tombstone : ev);
1707
+ // Activity is derived, so scrub it before the canonical commit. A crash here still has the complete retired fact
1708
+ // in its ledger/table; replay does not depend on activity rows. Nothing content-bearing remains after the rename.
1709
+ const erasedEvents = events.filter(ev => ev.id === target || (RETIREMENT_KINDS.has(ev.kind) && ev.target?.toUpperCase() === target));
1710
+ const activityRows = db.prepare(`SELECT id, summary FROM write_events
1711
+ WHERE note_path = '_memory_ledger_' AND action = ? AND created_at = ?
1712
+ AND (author IS NULL OR author = ?) AND (scope IS NULL OR scope = ?)`);
1713
+ const deleteActivity = db.prepare('DELETE FROM write_events WHERE id = ?');
1714
+ db.transaction(() => {
1715
+ for (const ev of erasedEvents) {
1716
+ const expected = ev.target !== undefined
1717
+ ? JSON.stringify({ id: `fact:${ev.target.toUpperCase()}`, title: `${ev.subject} ${ev.predicate}` })
1718
+ : `${ev.subject} ${ev.predicate} → ${ev.object}`;
1719
+ const rows = activityRows.all(ev.kind, ev.at, ev.source, ev.scope ?? 'global');
1720
+ for (const activity of rows) {
1721
+ // The erased event's own line, or that line with the facts it closed: never another line of the same length.
1722
+ const suffix = activity.summary.slice(expected.length);
1723
+ const isWriterSummary = activity.summary === expected
1724
+ || (activity.summary.startsWith(expected) && /^ \(closed facts? [0-9]+(?:, [0-9]+)*\)$/.test(suffix));
1725
+ if (isWriterSummary)
1726
+ deleteActivity.run(activity.id);
1727
+ }
1728
+ }
1729
+ })();
1730
+ const tmp = await (0, confinedWrite_1.createConfined)(vaultPath, `${rel}.erase-${crypto_1.default.randomUUID()}.tmp`, Buffer.from(LEDGER_HEADER + kept.map(ledgerEventBlock).join(''), 'utf8'));
1731
+ try {
1732
+ await promises_1.default.rename(tmp, file);
1733
+ }
1734
+ catch (err) {
1735
+ await promises_1.default.rm(tmp, { force: true });
1736
+ throw err;
1737
+ }
1738
+ // The rename is the commit point, so it reaches the disk before the index records the erasure as done: otherwise a power
1739
+ // loss could leave the old ledger, with the erased content, under an index that says it is gone.
1740
+ await (0, durableAppend_1.syncDirectory)(path_1.default.dirname(file));
1741
+ const ledgerChunkIds = ledgerChunks(db, file, rel);
1742
+ db.transaction(() => {
1743
+ // The Markdown index may have chunked the old ledger bytes. Drop that derived note atomically; the watcher may
1744
+ // index the content-free tombstone later, but no read can serve a stale subject/value chunk meanwhile.
1745
+ db.prepare('DELETE FROM notes WHERE path IN (?, ?)').run(file, rel);
1746
+ applyEvent(db, tombstone);
1747
+ const audit = db.prepare(`SELECT 1 FROM write_events WHERE note_path = '_retired_' AND action = 'delete' AND summary = 'deleted permanently: 1 fact' AND created_at = ?`).get(at);
1748
+ if (!audit)
1749
+ db.prepare(`INSERT INTO write_events (note_path, action, summary, created_at) VALUES ('_retired_', 'delete', 'deleted permanently: 1 fact', ?)`).run(at);
1750
+ })();
1751
+ for (const chunkId of ledgerChunkIds)
1752
+ vectorIndex?.delete(chunkId);
1753
+ return { ok: true, action: 'deleted', ids: [`fact:${target}`], freedBytes: 0, at };
1754
+ }));
1755
+ });
1756
+ }
1757
+ /**
1758
+ * The fact events `ev` closed when it was saved, read from the index: the ones on its subject, predicate and scope whose value
1759
+ * ended at its own time, and the one its closed_event names. Ids only; they go in its erasure tombstone.
1760
+ */
1761
+ function factsClosedBy(db, ev, subjectEntityId) {
1762
+ const query = (table) => `SELECT event_id FROM ${table} WHERE event_id IS NOT NULL AND event_id <> ? AND valid_to = ?
1763
+ AND ((subject_entity_id = ? AND predicate = ? AND scope = ?) OR event_id = ?)`;
1764
+ const args = [ev.id, ev.at, subjectEntityId ?? -1, ev.predicate, (0, scope_1.normalizeScope)(ev.scope), ev.closedEvent?.toUpperCase() ?? ''];
1765
+ const ids = new Set();
1766
+ for (const table of ['facts', 'retired_facts']) {
1767
+ for (const closed of db.prepare(query(table)).pluck().all(...args))
1768
+ ids.add(closed.toUpperCase());
1769
+ }
1770
+ return [...ids].sort();
1771
+ }
1772
+ function ledgerChunks(db, absolute, relative) {
1773
+ return db.prepare('SELECT id FROM chunks WHERE note_path IN (?, ?)').all(absolute, relative).map(chunk => chunk.id);
1774
+ }
1527
1775
  const FACT_ID_RE = /^fact:(mem_[A-Za-z0-9]+)$/i;
1528
1776
  /** The event id in a fact id (`fact:MEM_…`), or null for any other memory id. */
1529
1777
  function factEventId(id) {
@@ -1575,6 +1823,9 @@ async function restoreFact(db, vaultPath, ledgerRelPath, id, source, validateAft
1575
1823
  const target = factEventId(id);
1576
1824
  if (target === null)
1577
1825
  throw new LifecycleInputError('id', `not a fact id: ${JSON.stringify(id)} (a fact's id is fact: and its eventId)`, 'invalid_argument');
1826
+ if ((0, factTerms_1.prepareOnce)(db, 'SELECT 1 FROM erased_facts WHERE event_id = ?').get(target)) {
1827
+ throw new LifecycleInputError('id', 'that fact was permanently erased and cannot be restored', 'memory_not_found');
1828
+ }
1578
1829
  const row = (0, factTerms_1.prepareOnce)(db, `SELECT r.predicate, r.object_text, r.scope, e.canonical_name AS subject
1579
1830
  FROM retired_facts r JOIN entities e ON e.id = r.subject_entity_id WHERE r.event_id = ?`).get(target);
1580
1831
  if (!row) {
@@ -1689,22 +1940,6 @@ function noteReadInstant(db, scope) {
1689
1940
  const DEFAULT_CURRENT_SQL = `((source_path = '${exports.LIFECYCLE_NOTE_PATH}' AND valid_to IS NULL)`
1690
1941
  + ` OR ((source_path IS NULL OR source_path <> '${exports.LIFECYCLE_NOTE_PATH}')`
1691
1942
  + ` AND valid_from <= ? AND (valid_to IS NULL OR valid_to > ?)))`;
1692
- /**
1693
- * Current facts only by default; asOf gives point-in-time; includeHistory gives
1694
- * everything. The default read judges a lifecycle fact by open rows (see
1695
- * DEFAULT_CURRENT_SQL), so a backward clock step cannot hide the current value;
1696
- * a note-derived fact is judged at the wall clock clamped up to the newest
1697
- * workspace observation (workspaceReadInstant), so the same step cannot hide it
1698
- * either; asOf keeps the exact point-in-time window. Lifecycle-owned facts are
1699
- * authoritative: if a lifecycle entity exists for the name, its facts are
1700
- * returned; otherwise the generic (possibly note-derived) entity lookup is used.
1701
- *
1702
- * When a name still matches SEVERAL lifecycle entities (a split a pre-fix write path
1703
- * could create, which nothing reconciles in place), facts from ALL of them are
1704
- * returned. A read must not throw — a startup or dashboard read would become an outage
1705
- * — and it must not hide half the memory behind a silent pick, so the ambiguity is
1706
- * disclosed instead. New writes refuse the same state (LegacyIdentityAmbiguityError).
1707
- */
1708
1943
  function factsFor(db, entityName, opts = {}) {
1709
1944
  let entityIds = [];
1710
1945
  for (const alias of (0, queryHelpers_1.identityNames)(entityName)) {