renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -8,7 +8,7 @@ exports.dropEvidenceForStems = dropEvidenceForStems;
8
8
  const node_fs_1 = __importDefault(require("node:fs"));
9
9
  const node_crypto_1 = __importDefault(require("node:crypto"));
10
10
  const node_path_1 = __importDefault(require("node:path"));
11
- const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
11
+ const sqlite_1 = __importDefault(require("../db/sqlite"));
12
12
  /**
13
13
  * Content-addressed cache for the H10 second-pass evidence extraction.
14
14
  *
@@ -58,7 +58,7 @@ class EvidenceCache {
58
58
  failurePutStmt;
59
59
  constructor(dbPath) {
60
60
  node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
61
- this.db = new better_sqlite3_1.default(dbPath);
61
+ this.db = new sqlite_1.default(dbPath);
62
62
  this.db.pragma('journal_mode = WAL');
63
63
  this.db.pragma('busy_timeout = 10000');
64
64
  this.db.exec(`
@@ -166,7 +166,7 @@ exports.EvidenceCache = EvidenceCache;
166
166
  function dropEvidenceForStems(cachePath, stems) {
167
167
  if (stems.length === 0 || !node_fs_1.default.existsSync(cachePath))
168
168
  return;
169
- const db = new better_sqlite3_1.default(cachePath);
169
+ const db = new sqlite_1.default(cachePath);
170
170
  try {
171
171
  db.pragma('busy_timeout = 10000');
172
172
  if (!db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'evidence_cache'`).get())
@@ -5,6 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.resolveEvidenceBounds = resolveEvidenceBounds;
7
7
  exports.resolveEvidenceProvider = resolveEvidenceProvider;
8
+ exports.evidenceCachePath = evidenceCachePath;
9
+ exports.resolveEvidenceChat = resolveEvidenceChat;
8
10
  exports.resolveEvidenceToolOptions = resolveEvidenceToolOptions;
9
11
  exports.createEvidenceExtractor = createEvidenceExtractor;
10
12
  const node_path_1 = __importDefault(require("node:path"));
@@ -62,50 +64,69 @@ function resolveEvidenceProvider(env = process.env) {
62
64
  ].map(k => k.trim()).filter(Boolean);
63
65
  return { baseUrl, keys };
64
66
  }
65
- function resolveEvidenceToolOptions(config, env = process.env) {
66
- const mode = (0, gate_1.resolveEvidenceMode)(env);
67
- const model = env.RENOR_EVIDENCE_MODEL?.trim() || secondPass_1.EVIDENCE_MODEL_DEFAULT;
68
- const cachePath = env.RENOR_EVIDENCE_CACHE?.trim()
69
- || node_path_1.default.join(node_path_1.default.dirname(config.dbPath), 'evidence-cache.db');
70
- const bounds = resolveEvidenceBounds(env);
71
- // A workspace that opted in (RENOR_EVIDENCE_OPTIN, see optin.ts) reaches Ollama Cloud with the key file's key
72
- // alone, on gemma4:31b, and spends a daily allowance of calls. Unset, this is the resolver it always was.
67
+ /** Where the evidence pass caches, and where an opted-in workspace's daily allowance is counted: RENOR_EVIDENCE_CACHE, else beside the index. */
68
+ function evidenceCachePath(config, env = process.env) {
69
+ return env.RENOR_EVIDENCE_CACHE?.trim() || node_path_1.default.join(node_path_1.default.dirname(config.dbPath), 'evidence-cache.db');
70
+ }
71
+ /**
72
+ * The evidence pass's model and transport. A workspace that opted in (RENOR_EVIDENCE_OPTIN, see optin.ts) reaches Ollama
73
+ * Cloud with the key file's key alone, on gemma4:31b, and spends its daily allowance per HTTP attempt; otherwise the
74
+ * configured endpoint and keys (resolveEvidenceProvider) with RENOR_EVIDENCE_MODEL. No chat when no key is configured.
75
+ * `transport` bounds each call; unset, the transport's own defaults. Another reader of notes at query time (the answer
76
+ * check, mvp/answerCheck.ts) builds its chat here, so it sends them only where the evidence pass would.
77
+ */
78
+ function resolveEvidenceChat(config, env = process.env, transport = {}) {
73
79
  const optin = (0, optin_1.resolveEvidenceOptin)(env);
74
80
  const { baseUrl, keys } = resolveEvidenceProvider(env);
75
- // Coverage knob (RENOR_EVIDENCE_SESSIONS). Default 20 matches the
76
- // promotion-grade full-500 arm. One model call per uncached search,
77
- // regardless of session count; explicit values remain bounded.
78
- // A value that floors below 1 is treated as UNSET, not as "0 sessions". A
79
- // fractional setting is finite and positive, so a bare `> 0` test let it
80
- // through and `Math.floor` turned it into 0 — and 0 is not a silent no-op:
81
- // `loadEvidenceSessionsFromVault` rejects it (`maxSessions must be > 0`),
82
- // which the `evidence` MCP tool surfaces as a thrown error and the search path
83
- // swallows as `extractor-error`. Floor first, then require >= 1.
84
- const requested = Number(env.RENOR_EVIDENCE_SESSIONS ?? '');
85
- const requestedSessions = Number.isFinite(requested) ? Math.floor(requested) : Number.NaN;
86
- const maxSessions = Number.isFinite(requestedSessions) && requestedSessions >= 1
87
- ? Math.min(requestedSessions, 50)
88
- : secondPass_1.MAX_EVIDENCE_SESSIONS;
81
+ const bounds = {
82
+ ...(transport.timeoutMs !== undefined ? { timeoutMs: transport.timeoutMs } : {}),
83
+ ...(transport.maxRetries !== undefined ? { maxRetries: transport.maxRetries } : {}),
84
+ };
89
85
  let chat;
90
86
  if (optin) {
91
87
  const optinKey = (0, optin_1.optinEndpointAllowed)(baseUrl) ? (0, optin_1.readOptinKey)(optin.keyFile) : null;
92
88
  if (optinKey) {
93
89
  const dailyCap = optin.dailyCap;
90
+ const cachePath = evidenceCachePath(config, env);
94
91
  // The allowance is spent per HTTP attempt (a retried 429 or 5xx is another request to the provider), so it
95
92
  // sits inside the transport rather than around it.
96
93
  chat = (0, ollamaChat_1.createOllamaChat)({
97
94
  keys: [optinKey],
98
95
  beforeAttempt: (0, optin_1.dailyBudgetGate)(() => new optin_1.EvidenceDailyBudget(cachePath, dailyCap)),
96
+ ...bounds,
99
97
  });
100
98
  }
101
99
  }
102
100
  else if (keys.length > 0) {
103
- chat = (0, ollamaChat_1.createOllamaChat)({ baseUrl, keys });
101
+ chat = (0, ollamaChat_1.createOllamaChat)({ baseUrl, keys, ...bounds });
104
102
  }
103
+ // The opt-in is the gemma4:31b decision: a stray RENOR_EVIDENCE_MODEL does not change what the privacy page names.
104
+ const model = optin ? secondPass_1.EVIDENCE_MODEL_DEFAULT : env.RENOR_EVIDENCE_MODEL?.trim() || secondPass_1.EVIDENCE_MODEL_DEFAULT;
105
+ return { model, ...(chat ? { chat } : {}), optin };
106
+ }
107
+ function resolveEvidenceToolOptions(config, env = process.env) {
108
+ const mode = (0, gate_1.resolveEvidenceMode)(env);
109
+ const cachePath = evidenceCachePath(config, env);
110
+ const bounds = resolveEvidenceBounds(env);
111
+ // The model, the transport and the opt-in (key file, endpoint, gemma4:31b, daily allowance): resolveEvidenceChat.
112
+ const { model, chat, optin } = resolveEvidenceChat(config, env);
113
+ // Coverage knob (RENOR_EVIDENCE_SESSIONS). Default 20 matches the
114
+ // promotion-grade full-500 arm. One model call per uncached search,
115
+ // regardless of session count; explicit values remain bounded.
116
+ // A value that floors below 1 is treated as UNSET, not as "0 sessions". A
117
+ // fractional setting is finite and positive, so a bare `> 0` test let it
118
+ // through and `Math.floor` turned it into 0 — and 0 is not a silent no-op:
119
+ // `loadEvidenceSessionsFromVault` rejects it (`maxSessions must be > 0`),
120
+ // which the `evidence` MCP tool surfaces as a thrown error and the search path
121
+ // swallows as `extractor-error`. Floor first, then require >= 1.
122
+ const requested = Number(env.RENOR_EVIDENCE_SESSIONS ?? '');
123
+ const requestedSessions = Number.isFinite(requested) ? Math.floor(requested) : Number.NaN;
124
+ const maxSessions = Number.isFinite(requestedSessions) && requestedSessions >= 1
125
+ ? Math.min(requestedSessions, 50)
126
+ : secondPass_1.MAX_EVIDENCE_SESSIONS;
105
127
  return {
106
128
  mode,
107
- // The opt-in is the gemma4:31b decision: a stray RENOR_EVIDENCE_MODEL does not change what the privacy page names.
108
- model: optin ? secondPass_1.EVIDENCE_MODEL_DEFAULT : model,
129
+ model,
109
130
  maxSessions,
110
131
  charCap: secondPass_1.EVIDENCE_SESSION_CHAR_CAP,
111
132
  cachePath,
@@ -3,11 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.EvidenceExtractor = exports.EvidenceReplyError = exports.EVIDENCE_REASONING_EFFORTS = exports.EVIDENCE_SESSION_CHAR_CAP = exports.MAX_EVIDENCE_SESSIONS = exports.EVIDENCE_MODEL_DEFAULT = exports.EVIDENCE_PROMPT_TEMPLATE_SCOPED = exports.EVIDENCE_PROMPT_TEMPLATE_EXHAUSTIVE = exports.EVIDENCE_PROMPT_TEMPLATE = void 0;
7
- exports.evidenceEnumerateMode = evidenceEnumerateMode;
8
- exports.evidenceEnumerateEnabled = evidenceEnumerateEnabled;
9
- exports.scoreBodyOverlap = scoreBodyOverlap;
10
- exports.bodyRankEnabled = bodyRankEnabled;
6
+ exports.EvidenceExtractor = exports.EvidenceReplyError = exports.EVIDENCE_REASONING_EFFORTS = exports.EVIDENCE_SESSION_CHAR_CAP = exports.MAX_EVIDENCE_SESSIONS = exports.EVIDENCE_MODEL_DEFAULT = exports.EVIDENCE_PROMPT_TEMPLATE = void 0;
11
7
  exports.loadEvidenceSessionsFromVault = loadEvidenceSessionsFromVault;
12
8
  exports.renderEvidencePrompt = renderEvidencePrompt;
13
9
  exports.parseEvidenceItems = parseEvidenceItems;
@@ -47,95 +43,6 @@ relevant to answering the question. Rules:
47
43
  SESSIONS:
48
44
  {sessions}
49
45
 
50
- Respond with ONLY minified JSON, no markdown:
51
- {{"items":[{{"item":"...","action":"...","amount":"...","date":"..."}}]}}`;
52
- /**
53
- * EXHAUSTIVE enumeration variant of the evidence prompt (RENOR_EVIDENCE_ENUMERATE,
54
- * default OFF — opt in with 1/true/yes/on).
55
- *
56
- * Why this exists, measured: the shipped prompt asks for items "that could be
57
- * relevant to answering the question" — a relevance FILTER applied to an
58
- * enumeration task. On the champion's 23 count/aggregate wrongs that filter is
59
- * the binding constraint: the evidence table holds a mean of 5.3 items, and 13 of
60
- * the 23 have FEWER items than the gold count requires. The count-contract arm
61
- * proved the reader's arithmetic is correct — it lists 2 items and answers "2"
62
- * for gold 3 — so the items are simply absent from the table it counts.
63
- *
64
- * Prompt-variant A/B on one case (gpt4_31ff4165, "How many health-related devices
65
- * do I use in a day?", gold 4) at temperature 0:
66
- *
67
- * shipped prompt : 4 items, 134 completion tokens
68
- * this variant : 27 items, 936 completion tokens
69
- *
70
- * The variant asks for everything mentioned and states that completeness is the
71
- * only goal, that missing an item is a failure, and that including an irrelevant
72
- * item is NOT a failure — the exact inverse of the shipped relevance filter.
73
- *
74
- * Scope discipline: this changes the EVIDENCE prompt only. The frozen
75
- * judge/extractor model, the retrieval ranking, the schema, the item payload and
76
- * the fact/chronicle/preference lanes are untouched. Default OFF, so shipped
77
- * behaviour is byte-identical unless an eval arm opts in.
78
- */
79
- exports.EVIDENCE_PROMPT_TEMPLATE_EXHAUSTIVE = `You extract a COMPLETE inventory of items from a user's dated conversation history.
80
-
81
- QUESTION: {question}
82
-
83
- Below are the sessions (with dates) that a memory system retrieved for this question.
84
-
85
- TASK: List EVERY distinct item, device, object, purchase, place, person, activity, event or amount mentioned ANYWHERE in the sessions below — whether or not it appears relevant to the question. Completeness is the ONLY goal.
86
- Rules:
87
- - COMPLETENESS OVER BREVITY. If the sessions mention ten things, list all ten. Do not summarise, do not select, do not rank, do not judge relevance.
88
- - Missing an item is a failure. Including an item that turns out to be irrelevant is NOT a failure.
89
- - One entry per distinct item (e.g., each product, each device, each trip, each course, each amount).
90
- - Do NOT merge different items. Do NOT compute any totals.
91
- - "action" = what happened (bought / visited / completed / raised / inherited / used / ...).
92
- - "amount" = numeric value + unit when stated (e.g., "3 days", "$120", "100 points"); else "".
93
- - "date" = the session/mention date when available; else "".
94
-
95
- SESSIONS:
96
- {sessions}
97
-
98
- Respond with ONLY minified JSON, no markdown:
99
- {{"items":[{{"item":"...","action":"...","amount":"...","date":"..."}}]}}`;
100
- function evidenceEnumerateMode(env = process.env) {
101
- const raw = (env.RENOR_EVIDENCE_ENUMERATE ?? '').trim().toLowerCase();
102
- if (raw === 'scoped')
103
- return 'scoped';
104
- if (raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on')
105
- return 'all';
106
- return 'off';
107
- }
108
- /** Back-compat boolean form (true for either enumeration mode). */
109
- function evidenceEnumerateEnabled(env = process.env) {
110
- return evidenceEnumerateMode(env) !== 'off';
111
- }
112
- /**
113
- * SCOPED enumeration template: the model first names the entity type the question
114
- * counts, then lists every instance of that type. Deliberately instructs the model
115
- * to EXCLUDE the incidental magnitudes it is not being asked to count (steps per
116
- * day, hours of sleep, battery pack sizes) — the exact mass that made `all`
117
- * unusable. Gold-blind: the only input is the user's question and the retrieved
118
- * sessions.
119
- */
120
- exports.EVIDENCE_PROMPT_TEMPLATE_SCOPED = `You extract a COMPLETE inventory of one specific kind of entity from a user's dated conversation history.
121
-
122
- QUESTION: {question}
123
-
124
- Below are the sessions (with dates) that a memory system retrieved for this question.
125
-
126
- STEP 1 — From the QUESTION, decide the single kind of thing being counted or totalled. Examples: "how many items of clothing" -> garments; "how many devices do I use" -> devices; "how many albums did I buy" -> albums; "how much did I spend in total" -> purchases with an amount; "how many properties did I view" -> properties.
127
-
128
- STEP 2 — List EVERY instance of THAT kind found ANYWHERE in the sessions below. Completeness is the only goal.
129
- Rules:
130
- - Do NOT list incidental magnitudes of the same sentences. If counting devices, do NOT list "10,000 steps", "7-9 hours of sleep", "size 13 batteries" or "8-10 glasses of fluid" — those are not devices.
131
- - Do NOT summarise, do NOT select, do NOT rank, do NOT judge relevance.
132
- - Missing an instance of the counted kind is a failure. Including one that turns out to be irrelevant is NOT a failure.
133
- - Keep each entry short: the entity itself, plus the amount/detail if the question is about an amount.
134
- - Do NOT merge different instances. Do NOT compute any totals.
135
-
136
- SESSIONS:
137
- {sessions}
138
-
139
46
  Respond with ONLY minified JSON, no markdown:
140
47
  {{"items":[{{"item":"...","action":"...","amount":"...","date":"..."}}]}}`;
141
48
  exports.EVIDENCE_MODEL_DEFAULT = 'gemma4:31b';
@@ -143,52 +50,11 @@ exports.MAX_EVIDENCE_SESSIONS = 20;
143
50
  exports.EVIDENCE_SESSION_CHAR_CAP = 6000;
144
51
  /** The `reasoning_effort` values OpenAI-compatible providers share. Which ones a model accepts is the provider's call. */
145
52
  exports.EVIDENCE_REASONING_EFFORTS = ['none', 'minimal', 'low', 'medium', 'high'];
146
- /**
147
- * H9-family stop list (same shape as mvp/agentMemory.ts FTS_STOP_WORDS).
148
- */
149
- const BODY_RANK_STOP_WORDS = new Set([
150
- 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'what', 'which', 'who', 'where', 'when',
151
- 'why', 'how', 'do', 'does', 'did', 'to', 'for', 'of', 'in', 'on', 'at', 'with', 'and',
152
- 'or', 'my', 'your', 'our', 'their', 'his', 'her', 'it', 'this', 'that', 'about',
153
- ]);
154
- /** Unicode-aware lexical tokens: letter/number runs, 2..64 chars (H9 family). */
155
- function bodyRankTokens(text) {
156
- return (text.toLowerCase().match(/[\p{L}\p{N}]{2,64}/gu) ?? []).slice(0, 20000);
157
- }
158
- /**
159
- * Deterministic, gold-blind lexical query-term overlap between a query and a
160
- * candidate note body: total frequency of each distinct non-stop query term in
161
- * the body. Zero model calls and zero embeddings at query time.
162
- */
163
- function scoreBodyOverlap(query, body) {
164
- const queryTerms = new Set(bodyRankTokens(query).filter(term => !BODY_RANK_STOP_WORDS.has(term)));
165
- if (queryTerms.size === 0)
166
- return 0;
167
- const counts = new Map();
168
- for (const token of bodyRankTokens(body))
169
- counts.set(token, (counts.get(token) ?? 0) + 1);
170
- let score = 0;
171
- for (const term of queryTerms)
172
- score += counts.get(term) ?? 0;
173
- return score;
174
- }
175
- /** Opt-in kill-switch for body-relevance selection; default off. */
176
- function bodyRankEnabled(env = process.env) {
177
- const raw = (env.RENOR_EVIDENCE_BODY_RANK ?? '').trim().toLowerCase();
178
- return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
179
- }
180
53
  /**
181
54
  * Walk ranked hits as session identities, then hydrate each selected session
182
55
  * from the complete source note. Retrieved chunk text is never extractor input.
183
56
  * Unreadable/virtual/out-of-vault paths are skipped and later hits backfill.
184
- *
185
- * Default (RENOR_EVIDENCE_BODY_RANK off): the first maxSessions unique notes by
186
- * encounter rank. With RENOR_EVIDENCE_BODY_RANK=1: every unique candidate note
187
- * among the hits is hydrated (same ≤charCap truncation), scored by
188
- * `scoreBodyOverlap`, and the top maxSessions are selected by
189
- * (score desc, hit rank asc, notePath asc). Deterministic and gold-blind;
190
- * per-session hydrated text and therefore extraction prompt bytes are
191
- * unchanged, so the evidence content-addressed cache stays valid.
57
+ * The first maxSessions unique notes by encounter rank are selected.
192
58
  */
193
59
  async function loadEvidenceSessionsFromVault(hits, vaultPath, opts = {}) {
194
60
  const maxSessions = opts.maxSessions ?? exports.MAX_EVIDENCE_SESSIONS;
@@ -197,12 +63,10 @@ async function loadEvidenceSessionsFromVault(hits, vaultPath, opts = {}) {
197
63
  throw new Error('maxSessions must be > 0');
198
64
  if (charCap <= 0)
199
65
  throw new Error('charCap must be > 0');
200
- const bodyRank = opts.bodyRank ?? bodyRankEnabled();
201
66
  const root = node_path_1.default.resolve(vaultPath);
202
67
  const seen = new Set();
203
- const candidates = [];
204
- for (let hitRank = 0; hitRank < hits.length; hitRank++) {
205
- const hit = hits[hitRank];
68
+ const sessions = [];
69
+ for (const hit of hits) {
206
70
  if (!hit.notePath || seen.has(hit.notePath))
207
71
  continue;
208
72
  seen.add(hit.notePath);
@@ -224,33 +88,11 @@ async function loadEvidenceSessionsFromVault(hits, vaultPath, opts = {}) {
224
88
  if (!body)
225
89
  continue;
226
90
  const text = body.length > charCap ? `${body.slice(0, charCap)} …[truncated]` : body;
227
- candidates.push({ stem: hit.notePath, date: hit.session_date, text, hitRank });
228
- if (!bodyRank && candidates.length >= maxSessions)
91
+ sessions.push({ stem: hit.notePath, date: hit.session_date, text });
92
+ if (sessions.length >= maxSessions)
229
93
  break;
230
94
  }
231
- // Flag-off path must be byte-identical to shipped behavior — strip the
232
- // internal hitRank before returning (review finding P2-1).
233
- if (!bodyRank)
234
- return candidates.map(({ hitRank, ...session }) => session);
235
- return selectSessionsByBodyOverlap(opts.query ?? '', candidates).slice(0, maxSessions);
236
- }
237
- /**
238
- * Body-relevance selection: score desc, then hit rank asc, then notePath asc.
239
- * Fully deterministic; a zero-score query degrades to encounter order.
240
- */
241
- function selectSessionsByBodyOverlap(query, candidates) {
242
- const scored = candidates.map(candidate => ({
243
- candidate,
244
- score: scoreBodyOverlap(query, candidate.text),
245
- }));
246
- scored.sort((a, b) => b.score - a.score
247
- || a.candidate.hitRank - b.candidate.hitRank
248
- || (a.candidate.stem < b.candidate.stem ? -1 : a.candidate.stem > b.candidate.stem ? 1 : 0));
249
- return scored.map(({ candidate }) => ({
250
- stem: candidate.stem,
251
- date: candidate.date,
252
- text: candidate.text,
253
- }));
95
+ return sessions;
254
96
  }
255
97
  function sessionBlock(session) {
256
98
  const date = session.date ?? '';
@@ -258,11 +100,7 @@ function sessionBlock(session) {
258
100
  }
259
101
  /** Exact prompt the extractor receives; the cache keys on this string. */
260
102
  function renderEvidencePrompt(query, sessions) {
261
- const mode = evidenceEnumerateMode();
262
- const template = mode === 'scoped' ? exports.EVIDENCE_PROMPT_TEMPLATE_SCOPED
263
- : mode === 'all' ? exports.EVIDENCE_PROMPT_TEMPLATE_EXHAUSTIVE
264
- : exports.EVIDENCE_PROMPT_TEMPLATE;
265
- return template
103
+ return exports.EVIDENCE_PROMPT_TEMPLATE
266
104
  .replace('{question}', query)
267
105
  .replace('{sessions}', sessions.map(sessionBlock).join('\n\n'));
268
106
  }
@@ -42,5 +42,5 @@ Rules:
42
42
  if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.facts)) {
43
43
  throw new Error('Fact extraction response is missing a facts array');
44
44
  }
45
- return parsed.facts.filter((f) => f.subject && f.predicate && f.object && typeof f.confidence === 'number');
45
+ return parsed.facts.filter((f) => Boolean(f.subject && f.predicate && f.object && typeof f.confidence === 'number'));
46
46
  }
@@ -106,10 +106,7 @@ async function extractPreferences(client, noteTitle, noteBody) {
106
106
  // `statement` are dropped below, so mis-shaped rows can never become false
107
107
  // preferences — empty is the worst case.
108
108
  const firstArray = Object.values(parsed).find((v) => Array.isArray(v) && v.length > 0 && typeof v[0] === 'object' && v[0] !== null);
109
- if (firstArray)
110
- parsed.prefs = firstArray;
111
- else
112
- parsed.prefs = [parsed];
109
+ parsed.prefs = firstArray ?? [parsed];
113
110
  }
114
111
  if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.prefs)) {
115
112
  throw new typedFacts_1.ExtractorParseError('extractor-parse-error: preference extraction response is missing a prefs array');
@@ -195,7 +195,7 @@ async function extractNumericFacts(client, noteTitle, noteBody) {
195
195
  continue; // malformed rows are dropped, not fatal
196
196
  const fact = { value, object };
197
197
  for (const key of ['unit', 'verb', 'event']) {
198
- const v = raw[key];
198
+ const v = raw?.[key];
199
199
  if (v !== undefined && v !== null && String(v).trim())
200
200
  fact[key] = String(v).trim();
201
201
  }
@@ -14,6 +14,7 @@ const schema_1 = require("../db/schema");
14
14
  const hasher_1 = require("../indexer/hasher");
15
15
  const vectorIndex_1 = require("../embeddings/vectorIndex");
16
16
  const confinedWrite_1 = require("../memory/confinedWrite");
17
+ const durableAppend_1 = require("../memory/durableAppend");
17
18
  const lifecycle_1 = require("../memory/lifecycle");
18
19
  const secretGuard_1 = require("../extraction/secretGuard");
19
20
  const secretScreen_1 = require("./secretScreen");
@@ -70,6 +71,7 @@ async function applyImportPlan(plan, _bundle, options) {
70
71
  const created = { files: [], dirs: [] };
71
72
  let phase = 'notes';
72
73
  let priorLedger = null;
74
+ let appendedLedger = null;
73
75
  // Set immediately before this invocation's first append. A failed read or parse must
74
76
  // never enter ledger compensation: no ledger bytes have been touched yet.
75
77
  let ledgerTouched = false;
@@ -168,19 +170,28 @@ async function applyImportPlan(plan, _bundle, options) {
168
170
  // directories that are missing now, so a rollback cannot leave an empty tree behind.
169
171
  for (const dir of missingDirsUnder(memoryDir, path_1.default.dirname(ledgerPath)))
170
172
  created.dirs.push(dir);
171
- for (const event of plan.events) {
172
- const eventKey = event.id.toUpperCase();
173
- if (existingLedgerIds.has(eventKey)) {
174
- eventsAlreadyPresent += 1;
175
- continue;
173
+ try {
174
+ for (const event of plan.events) {
175
+ const eventKey = event.id.toUpperCase();
176
+ if (existingLedgerIds.has(eventKey)) {
177
+ eventsAlreadyPresent += 1;
178
+ continue;
179
+ }
180
+ // From this point the durable bytes may change, so and only so can rollback restore
181
+ // the prior snapshot (or remove a ledger this invocation created from an absent path).
182
+ phase = 'ledger';
183
+ ledgerTouched = true;
184
+ await (0, lifecycle_1.appendLedgerEvent)(memoryDir, ledgerRelPath, toLedgerEvent(event));
185
+ existingLedgerIds.add(eventKey);
186
+ eventsAppended += 1;
176
187
  }
177
- // From this point the durable bytes may change, so and only so can rollback restore
178
- // the prior snapshot (or remove a ledger this invocation created from an absent path).
179
- phase = 'ledger';
180
- ledgerTouched = true;
181
- await (0, lifecycle_1.appendLedgerEvent)(memoryDir, ledgerRelPath, toLedgerEvent(event));
182
- existingLedgerIds.add(eventKey);
183
- eventsAppended += 1;
188
+ }
189
+ finally {
190
+ // Capture what this run actually left behind even when an append rejects. The
191
+ // compare-before-restore guard must compare against these bytes, not `null`, after
192
+ // an earlier event in the same plan was appended successfully.
193
+ if (ledgerTouched)
194
+ appendedLedger = readFileOrNull(ledgerPath);
184
195
  }
185
196
  phase = 'index';
186
197
  db = (0, client_1.openDb)(dbPath);
@@ -248,7 +259,7 @@ async function applyImportPlan(plan, _bundle, options) {
248
259
  // did not create — unchanged notes, conflicts, the operator's own files — are never
249
260
  // touched, and pruning is limited to directories that are empty afterwards.
250
261
  const restore = ledgerTouched
251
- ? restoreLedger(ledgerPath, priorLedger)
262
+ ? await restoreLedger(ledgerPath, priorLedger, appendedLedger)
252
263
  : { restored: false };
253
264
  // A ledger that could not be restored still holds its carried events, and those events
254
265
  // are canonical: removing the notes they describe would leave a ledger pointing at
@@ -484,8 +495,25 @@ function readFileOrNull(filePath) {
484
495
  * Returns the OUTCOME rather than swallowing it: a run that could not restore the ledger
485
496
  * still holds the carried events, and must say so instead of claiming a clean rollback.
486
497
  */
487
- function restoreLedger(ledgerPath, prior) {
498
+ function restoreLedger(ledgerPath, prior, appended) {
499
+ // Under the lock every ledger writer takes (appends, and the erasure's rewrite in lifecycle.ts), so no writer can
500
+ // change the ledger between the checks below and the restore.
501
+ return (0, durableAppend_1.withAppendLock)(ledgerPath, async () => restoreLedgerLocked(ledgerPath, prior, appended));
502
+ }
503
+ function restoreLedgerLocked(ledgerPath, prior, appended) {
488
504
  try {
505
+ // Compensation owns only the bytes it appended. If another lifecycle writer changed the
506
+ // ledger afterwards (especially by replacing a fact with an erase tombstone), restoring
507
+ // this import's older snapshot would resurrect that writer's deleted canonical content.
508
+ if (readFileOrNull(ledgerPath) !== appended) {
509
+ return { restored: false, error: 'the ledger changed while this import ran (another save, or an erasure); refusing to overwrite newer canonical memory' };
510
+ }
511
+ // The snapshot was read before the appends, without the lock. Only appends may have happened since: then the snapshot
512
+ // is a prefix of what this run left. Anything else (an erasure rewrote the ledger in between) makes the snapshot stale,
513
+ // and restoring it would write erased content back.
514
+ if (prior !== null && (appended === null || !appended.startsWith(prior))) {
515
+ return { restored: false, error: 'the ledger changed while this import ran (another save, or an erasure); refusing to overwrite newer canonical memory' };
516
+ }
489
517
  if (prior === null) {
490
518
  if (fs_1.default.existsSync(ledgerPath))
491
519
  fs_1.default.unlinkSync(ledgerPath);
@@ -325,6 +325,7 @@ async function run(argv) {
325
325
  warnings,
326
326
  };
327
327
  let exitCode = 0;
328
+ let applied;
328
329
  if (options.apply) {
329
330
  const result = await (0, apply_1.applyImportPlan)(plan, bundle, {
330
331
  memoryDir,
@@ -332,7 +333,7 @@ async function run(argv) {
332
333
  ledgerRelPath: options.ledgerRel,
333
334
  canonicalOnly: options.canonicalOnly,
334
335
  });
335
- summary.applied = result;
336
+ summary.applied = applied = result;
336
337
  summary.counts = {
337
338
  ...summary.counts,
338
339
  eventsAlreadyInDestination: result.eventsAlreadyPresent,
@@ -365,29 +366,29 @@ async function run(argv) {
365
366
  console.log(` ! ${plan.counts.chunksNotLocated} chunk(s) not locatable in the carried bytes; ${plan.counts.chunksSpanInherited} inherited the source span, the rest have an unknown span`);
366
367
  }
367
368
  console.log(` ledger events : ${plan.counts.eventsToAppend} new`
368
- + (options.apply ? ` (${summary.applied.eventsAppended} appended, ${summary.applied.eventsAlreadyPresent} already present)` : ''));
369
+ + (applied ? ` (${applied.eventsAppended} appended, ${applied.eventsAlreadyPresent} already present)` : ''));
369
370
  console.log(` facts : ${plan.counts.factsInBundle} current in source, ${plan.counts.eventsToAppend} carried, `
370
371
  + `${plan.counts.multiValuedFactsDropped} multi-valued dropped, ${plan.counts.factsSkipped} other skips`);
371
372
  for (const skip of plan.skipped.events) {
372
373
  console.log(` - ${skip.count} × ${skip.reason}`);
373
374
  }
374
- if (options.apply) {
375
- const applied = summary.applied;
375
+ if (applied) {
376
+ const dest = applied.destination;
376
377
  console.log(` files : ${applied.notesWritten} written, ${applied.notesUnchanged} already identical, ${applied.notesConflicting} conflicting (never overwritten or indexed)`);
377
- for (const conflict of (applied.conflicts ?? []).slice(0, 10)) {
378
+ for (const conflict of applied.conflicts.slice(0, 10)) {
378
379
  console.log(` ! conflict (left as-is, not indexed): ${conflict}`);
379
380
  }
380
- if ((applied.conflicts ?? []).length > 10) {
381
+ if (applied.conflicts.length > 10) {
381
382
  console.log(` ! …and ${applied.conflicts.length - 10} more`);
382
383
  }
383
384
  console.log(` index rows : ${applied.indexRows.notesInserted} notes, ${applied.indexRows.chunksInserted} chunks, ${applied.indexRows.entitiesInserted} entities inserted`);
384
385
  console.log(` replayed : ${applied.ledgerReplayed} ledger event(s) applied to the index`);
385
- console.log(` workspace now : notes ${applied.destination.notes} · chunks ${applied.destination.chunks} · entities ${applied.destination.entities} · facts ${applied.destination.facts} (${applied.destination.currentFacts} current)`);
386
+ console.log(` workspace now : notes ${dest?.notes} · chunks ${dest?.chunks} · entities ${dest?.entities} · facts ${dest?.facts} (${dest?.currentFacts} current)`);
386
387
  }
387
388
  for (const warning of warnings)
388
389
  console.log(` note : ${warning}`);
389
390
  }
390
- if (options.apply && summary.applied.notesConflicting > 0)
391
+ if (applied && applied.notesConflicting > 0)
391
392
  exitCode = 3;
392
393
  // Like a conflict, a note the guard left out is something the operator asked for that was not imported.
393
394
  const guardSkips = plan.skipped.secretGuard;
@@ -38,6 +38,8 @@ function readExistingStreams(dbPath, ledgerPath) {
38
38
  try {
39
39
  const content = fs_1.default.readFileSync(ledgerPath, 'utf8');
40
40
  for (const event of (0, lifecycle_1.parseLedgerEvents)(content)) {
41
+ if (event.kind === 'erase')
42
+ continue; // an erasure tombstone keeps no subject or predicate: it declares no stream
41
43
  streams.add((0, plan_1.streamKey)(event.subject, event.predicate, event.scope ?? 'global'));
42
44
  }
43
45
  }
@@ -3,7 +3,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.EVENT_ID_RE = void 0;
7
6
  exports.deterministicEventId = deterministicEventId;
8
7
  exports.toCanonicalInstant = toCanonicalInstant;
9
8
  exports.tryNormalizeScope = tryNormalizeScope;
@@ -29,7 +28,6 @@ const scope_1 = require("../mvp/scope");
29
28
  * here: a validator that drifts looser fails at boot, and one that drifts stricter
30
29
  * silently loses memory.
31
30
  */
32
- exports.EVENT_ID_RE = /^mem_[A-Za-z0-9]+$/;
33
31
  /** Field limits as enforced by `lifecycle.ts` `validateField`. */
34
32
  const FIELD_LIMITS = { subject: 512, predicate: 256, object: 8192, source: 256 };
35
33
  const CONTROL_CHARS = /[\u0000-\u001F\u007F]/;
@@ -4,7 +4,6 @@ exports.IMPORT_ROOT = void 0;
4
4
  exports.streamKey = streamKey;
5
5
  exports.assertImportNamespace = assertImportNamespace;
6
6
  exports.buildImportPlan = buildImportPlan;
7
- exports.contentHash = contentHash;
8
7
  const hasher_1 = require("../indexer/hasher");
9
8
  const queryHelpers_1 = require("../retrieval/queryHelpers");
10
9
  const orchestrator_1 = require("../retrieval/orchestrator");
@@ -200,7 +199,3 @@ function buildImportPlan(bundle, options) {
200
199
  },
201
200
  };
202
201
  }
203
- /** The engine's own note change-detection hash, re-exported for callers. */
204
- function contentHash(content) {
205
- return (0, hasher_1.hashContent)(content);
206
- }
@@ -306,7 +306,7 @@ function readBundleSource(filePath, filter = { include: [], exclude: [] }) {
306
306
  sourcePath: String(record.sourcePath ?? ''),
307
307
  origin: record.origin === 'ledger' ? 'ledger' : 'note',
308
308
  eventId: typeof record.eventId === 'string' ? record.eventId : undefined,
309
- closedEvent: record.closedEvent ?? null,
309
+ closedEvent: (record.closedEvent ?? null), // carried unvalidated, as before
310
310
  ledgerSource: typeof record.ledgerSource === 'string' ? record.ledgerSource : undefined,
311
311
  });
312
312
  continue;
@@ -129,7 +129,10 @@ function parseNoteDetailed(filePath, content) {
129
129
  // the memory map's note view, the evidence pass). A daily note keeps a line of the user's words that would
130
130
  // start like an entry behind a backslash; the words are restored here, so none of them sees the escape.
131
131
  const text = (0, dailyNote_1.isDailyNotePath)(filePath) ? (0, dailyNote_1.decodeDailyNoteText)(content) : content;
132
- const { data: frontmatter, content: rawBody } = (0, gray_matter_1.default)(text, MATTER_OPTIONS);
132
+ const { data, content: rawBody } = (0, gray_matter_1.default)(text, MATTER_OPTIONS);
133
+ // Only a mapping is front matter. YAML also parses a list, a string, a number or a date as a document, and passing
134
+ // one on made the export write a list's items as keys `0`, `1`, ...
135
+ const frontmatter = Object.prototype.toString.call(data) === '[object Object]' ? data : {};
133
136
  assertFrontmatterBounded(frontmatter, content.length);
134
137
  const heading = rawBody.match(/^#\s+(.+)$/m);
135
138
  const title = heading?.[1].trim() ?? path_1.default.basename(filePath, '.md');