renor-engine 0.0.0-stage → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +75 -0
  3. package/README.md +105 -2
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +6 -0
  6. package/dist/api/apiAuth.js +43 -0
  7. package/dist/api/server.js +174 -0
  8. package/dist/buildIdentity.js +149 -0
  9. package/dist/callPhases.js +82 -0
  10. package/dist/cli-extractors.js +46 -0
  11. package/dist/cli.js +500 -0
  12. package/dist/config.js +65 -0
  13. package/dist/context/composer.js +320 -0
  14. package/dist/context/readHonesty.js +478 -0
  15. package/dist/context/rowStore.js +238 -0
  16. package/dist/conversation/formatter.js +81 -0
  17. package/dist/conversation/parser.js +193 -0
  18. package/dist/conversation/types.js +2 -0
  19. package/dist/conversation/writer.js +42 -0
  20. package/dist/db/client.js +149 -0
  21. package/dist/db/handoffTable.js +31 -0
  22. package/dist/db/indexSalvage.js +101 -0
  23. package/dist/db/migrations.js +670 -0
  24. package/dist/db/readCache.js +109 -0
  25. package/dist/db/schema.js +5 -0
  26. package/dist/db/sqlite.js +396 -0
  27. package/dist/db/statements.js +89 -0
  28. package/dist/digests/digestCache.js +146 -0
  29. package/dist/digests/digests.js +597 -0
  30. package/dist/digests/worker.js +350 -0
  31. package/dist/embeddings/caching.js +337 -0
  32. package/dist/embeddings/coalescing.js +177 -0
  33. package/dist/embeddings/interactive.js +103 -0
  34. package/dist/embeddings/local.js +70 -0
  35. package/dist/embeddings/openai.js +106 -0
  36. package/dist/embeddings/rebuild.js +166 -0
  37. package/dist/embeddings/simdScan.js +260 -0
  38. package/dist/embeddings/space.js +21 -0
  39. package/dist/embeddings/types.js +14 -0
  40. package/dist/embeddings/vectorIndex.js +674 -0
  41. package/dist/embeddings/vertex.js +334 -0
  42. package/dist/eval/artifacts.js +42 -0
  43. package/dist/eval/cacheReport.js +146 -0
  44. package/dist/eval/comparison.js +182 -0
  45. package/dist/eval/context700-candidate.js +145 -0
  46. package/dist/eval/evalCache.js +568 -0
  47. package/dist/eval/hashEmbedding.js +35 -0
  48. package/dist/eval/indexSnapshots.js +548 -0
  49. package/dist/eval/longmemeval.js +56 -0
  50. package/dist/eval/metrics.js +181 -0
  51. package/dist/eval/pageDump.js +67 -0
  52. package/dist/eval/renorbenchBaseline.js +486 -0
  53. package/dist/eval/renorbenchDocuments.js +134 -0
  54. package/dist/eval/renorbenchProduct.js +182 -0
  55. package/dist/eval/runLedger.js +279 -0
  56. package/dist/eval/runManifest.js +485 -0
  57. package/dist/eval/scoring.js +197 -0
  58. package/dist/eval/textTokens.js +17 -0
  59. package/dist/evidence/attach.js +264 -0
  60. package/dist/evidence/background.js +147 -0
  61. package/dist/evidence/evidenceCache.js +182 -0
  62. package/dist/evidence/gate.js +37 -0
  63. package/dist/evidence/keyPool.js +85 -0
  64. package/dist/evidence/ollamaChat.js +84 -0
  65. package/dist/evidence/optin.js +165 -0
  66. package/dist/evidence/options.js +151 -0
  67. package/dist/evidence/secondPass.js +294 -0
  68. package/dist/extraction/classifyCapture.js +34 -0
  69. package/dist/extraction/decision-model.js +174 -0
  70. package/dist/extraction/eventExtractor.js +113 -0
  71. package/dist/extraction/extractionClient.js +122 -0
  72. package/dist/extraction/factExtractor.js +46 -0
  73. package/dist/extraction/factsWritePath.js +179 -0
  74. package/dist/extraction/preferenceExtractor.js +150 -0
  75. package/dist/extraction/secretGuard.js +176 -0
  76. package/dist/extraction/secretScanner.js +665 -0
  77. package/dist/extraction/typedFacts.js +224 -0
  78. package/dist/extraction/writePathAdjudication.js +110 -0
  79. package/dist/importing/apply.js +672 -0
  80. package/dist/importing/bundle-write.js +80 -0
  81. package/dist/importing/cli.js +408 -0
  82. package/dist/importing/existing.js +51 -0
  83. package/dist/importing/facts.js +121 -0
  84. package/dist/importing/forgetImported.js +79 -0
  85. package/dist/importing/hosted.js +467 -0
  86. package/dist/importing/journalLog.js +96 -0
  87. package/dist/importing/markdown.js +98 -0
  88. package/dist/importing/plan.js +201 -0
  89. package/dist/importing/secretScreen.js +113 -0
  90. package/dist/importing/serialize.js +188 -0
  91. package/dist/importing/source.js +463 -0
  92. package/dist/importing/types.js +6 -0
  93. package/dist/indexer/chunker.js +319 -0
  94. package/dist/indexer/dailyEntries.js +63 -0
  95. package/dist/indexer/hasher.js +10 -0
  96. package/dist/indexer/laneRuns.js +48 -0
  97. package/dist/indexer/o200k.js +481 -0
  98. package/dist/indexer/o200k.json +8 -0
  99. package/dist/indexer/outsideRoot.js +77 -0
  100. package/dist/indexer/parser.js +170 -0
  101. package/dist/indexer/pipeline.js +988 -0
  102. package/dist/indexer/staleness.js +201 -0
  103. package/dist/indexer/tokenizer.js +77 -0
  104. package/dist/indexer/walker.js +170 -0
  105. package/dist/mcp/hive.js +88 -0
  106. package/dist/mcp/requestErrors.js +47 -0
  107. package/dist/mcp/server.js +426 -0
  108. package/dist/memory/agentLease.js +43 -0
  109. package/dist/memory/confinedWrite.js +315 -0
  110. package/dist/memory/controls.js +73 -0
  111. package/dist/memory/dailyNote.js +265 -0
  112. package/dist/memory/durableAppend.js +57 -0
  113. package/dist/memory/factConflicts.js +203 -0
  114. package/dist/memory/factTerms.js +321 -0
  115. package/dist/memory/handoff.js +363 -0
  116. package/dist/memory/hiveNameHeader.cjs +10 -0
  117. package/dist/memory/inventory.js +55 -0
  118. package/dist/memory/invisibleText.js +20 -0
  119. package/dist/memory/lifecycle.js +1991 -0
  120. package/dist/memory/limits.js +13 -0
  121. package/dist/memory/requests.js +183 -0
  122. package/dist/memory/sourceName.js +276 -0
  123. package/dist/memory/statusFacts.js +61 -0
  124. package/dist/memory/whatsNew.js +158 -0
  125. package/dist/memory/writer.js +175 -0
  126. package/dist/memory/writerConflicts.js +85 -0
  127. package/dist/memorymap/memoryMap.js +1594 -0
  128. package/dist/memorymap/replacedJournal.js +58 -0
  129. package/dist/memorymap/replacements.js +410 -0
  130. package/dist/mvp/activity.js +86 -0
  131. package/dist/mvp/agentMemory.js +1800 -0
  132. package/dist/mvp/answerCheck.js +211 -0
  133. package/dist/mvp/auth.js +531 -0
  134. package/dist/mvp/cli.js +75 -0
  135. package/dist/mvp/client-store.js +462 -0
  136. package/dist/mvp/compileCache.js +242 -0
  137. package/dist/mvp/compileSeed.js +33 -0
  138. package/dist/mvp/config.js +288 -0
  139. package/dist/mvp/connectionStatus.js +80 -0
  140. package/dist/mvp/contextDistinct.js +53 -0
  141. package/dist/mvp/contextProfile.js +87 -0
  142. package/dist/mvp/contextRelevance.js +413 -0
  143. package/dist/mvp/directorySurface.js +68 -0
  144. package/dist/mvp/engine.js +1078 -0
  145. package/dist/mvp/exportAll.js +374 -0
  146. package/dist/mvp/factLanePack.js +77 -0
  147. package/dist/mvp/factMarkers.js +65 -0
  148. package/dist/mvp/factsLane.js +203 -0
  149. package/dist/mvp/hiddenSpans.js +585 -0
  150. package/dist/mvp/history.js +414 -0
  151. package/dist/mvp/hitDates.js +31 -0
  152. package/dist/mvp/hitFloor.js +36 -0
  153. package/dist/mvp/identity.js +18 -0
  154. package/dist/mvp/idle.js +113 -0
  155. package/dist/mvp/log.js +118 -0
  156. package/dist/mvp/memoryDir.js +98 -0
  157. package/dist/mvp/posthogMcp.js +210 -0
  158. package/dist/mvp/readScope.js +104 -0
  159. package/dist/mvp/readUsefulness.js +222 -0
  160. package/dist/mvp/sanitize.js +48 -0
  161. package/dist/mvp/schemaRejections.js +91 -0
  162. package/dist/mvp/scope.js +127 -0
  163. package/dist/mvp/server.js +2604 -0
  164. package/dist/mvp/serverCard.js +38 -0
  165. package/dist/mvp/storedSize.js +139 -0
  166. package/dist/mvp/telemetry.js +474 -0
  167. package/dist/net/poolFetch.js +174 -0
  168. package/dist/recall/actionRecall.js +282 -0
  169. package/dist/recall/autoRecall.js +447 -0
  170. package/dist/recall/searchRecall.js +189 -0
  171. package/dist/retrieval/adviceLines.js +195 -0
  172. package/dist/retrieval/adviceQuestion.js +44 -0
  173. package/dist/retrieval/conversationShape.js +44 -0
  174. package/dist/retrieval/dailyEntryTime.js +55 -0
  175. package/dist/retrieval/entityLookup.js +162 -0
  176. package/dist/retrieval/factsFirst.js +188 -0
  177. package/dist/retrieval/keyword.js +84 -0
  178. package/dist/retrieval/ledgerEventTime.js +51 -0
  179. package/dist/retrieval/ledgerLiveness.js +83 -0
  180. package/dist/retrieval/ledgerPresentation.js +161 -0
  181. package/dist/retrieval/noteIdentity.js +57 -0
  182. package/dist/retrieval/orchestrator.js +1171 -0
  183. package/dist/retrieval/plainWording.js +236 -0
  184. package/dist/retrieval/queryHelpers.js +142 -0
  185. package/dist/retrieval/questionSpan.js +475 -0
  186. package/dist/retrieval/readerEvents.js +690 -0
  187. package/dist/retrieval/readerLayout.js +422 -0
  188. package/dist/retrieval/relativeTimeHints.js +113 -0
  189. package/dist/retrieval/semantic.js +67 -0
  190. package/dist/router/routerCache.js +106 -0
  191. package/dist/router/sessionRouter.js +726 -0
  192. package/dist/toolRefusal.js +120 -0
  193. package/dist/watcher/watcher.js +77 -0
  194. package/package.json +118 -4
@@ -0,0 +1,988 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.NoteTooLargeError = exports.MAX_NOTE_BYTES = void 0;
7
+ exports.resolveWikilinks = resolveWikilinks;
8
+ exports.indexFile = indexFile;
9
+ exports.indexVault = indexVault;
10
+ exports.removeDailyHeaderFacts = removeDailyHeaderFacts;
11
+ const promises_1 = __importDefault(require("fs/promises"));
12
+ const path_1 = __importDefault(require("path"));
13
+ const fs_1 = require("fs");
14
+ const crypto_1 = require("crypto");
15
+ const buffer_1 = require("buffer");
16
+ const statements_1 = require("../db/statements");
17
+ const hasher_1 = require("./hasher");
18
+ const dailyEntries_1 = require("./dailyEntries");
19
+ const parser_1 = require("./parser");
20
+ const chunker_1 = require("./chunker");
21
+ const walker_1 = require("./walker");
22
+ const space_1 = require("../embeddings/space");
23
+ const writer_1 = require("../memory/writer");
24
+ const fs_2 = require("fs");
25
+ const dailyNote_1 = require("../memory/dailyNote");
26
+ // HARMONIZED write path (merge of feat/c2-facts-writepath × origin/main):
27
+ // the fleet's extractors (typedFacts/eventExtractor/preferenceExtractor) are
28
+ // the CANDIDATE GENERATION layer (RENOR_TYPED_FACTS / RENOR_CHRONICLE /
29
+ // RENOR_PREFS, all default off); this branch's DecisionModel is the DECISION
30
+ // layer (capture-gate / dedupe / sensitivity-guardrail adjudication before
31
+ // any row is written). Single write path: decisions before writes.
32
+ const factsWritePath_1 = require("../extraction/factsWritePath");
33
+ const writePathAdjudication_1 = require("../extraction/writePathAdjudication");
34
+ const typedFacts_1 = require("../extraction/typedFacts");
35
+ const eventExtractor_1 = require("../extraction/eventExtractor");
36
+ const preferenceExtractor_1 = require("../extraction/preferenceExtractor");
37
+ const rowStore_1 = require("../context/rowStore");
38
+ const laneRuns_1 = require("./laneRuns");
39
+ const log_1 = require("../mvp/log");
40
+ const log = (0, log_1.createLogger)('engine');
41
+ /** Texts per request the embedding service takes at once (what Vertex's predict accepts). */
42
+ const EMBED_REQUEST_TEXTS = 32;
43
+ /**
44
+ * The largest note the indexer reads. A note is held whole, parsed, serialised and chunked in memory; a flat front matter
45
+ * costs about 28x its size in RSS (8 MB -> 487 MB, 64 MB -> 1.9 GB), and nothing else bounds a file on disk. A note over
46
+ * it is reported, never read (IndexResult.skippedTooLarge); split it, or move it out of the memory folder.
47
+ */
48
+ exports.MAX_NOTE_BYTES = 8 * 1024 * 1024;
49
+ /** A note over MAX_NOTE_BYTES. indexFile throws it before reading a byte; indexVault reports it instead of counting an error. */
50
+ class NoteTooLargeError extends Error {
51
+ filePath;
52
+ bytes;
53
+ constructor(filePath, bytes) {
54
+ super(`note is ${bytes} bytes; a note over ${exports.MAX_NOTE_BYTES / (1024 * 1024)} MiB (${exports.MAX_NOTE_BYTES} bytes) is not indexed. Split it or move it out of the memory folder.`);
55
+ this.filePath = filePath;
56
+ this.bytes = bytes;
57
+ this.name = 'NoteTooLargeError';
58
+ }
59
+ }
60
+ exports.NoteTooLargeError = NoteTooLargeError;
61
+ const FACT_PREDICATES = ['owner', 'status', 'updated', 'tags'];
62
+ /** A front-matter value as fact text: a scalar as written, a mapping (or binary) as JSON. Never throws, never "[object Object]". */
63
+ const factText = (v) => (v !== null && typeof v === 'object' && !(v instanceof Date) ? JSON.stringify(v) : String(v));
64
+ /** The note is indexed anyway; say which front-matter value was not text, so a note that seems to ignore it can be explained. */
65
+ function warnFrontmatter(filePath, what) {
66
+ log.warn('frontmatter_value_ignored', { note: path_1.default.basename(filePath), detail: `${what}; the note is indexed without it` });
67
+ }
68
+ /** The note's `kind`: a string, else none. Anything else (a mapping, a list, a date, a number) is ignored with a warning. */
69
+ function noteKind(filePath, frontmatter) {
70
+ const kind = frontmatter.kind;
71
+ if (typeof kind === 'string')
72
+ return kind;
73
+ if (kind !== undefined && kind !== null)
74
+ warnFrontmatter(filePath, 'front matter "kind" is not text');
75
+ return null;
76
+ }
77
+ /**
78
+ * The extraction version each structured-row lane writes today. This is the
79
+ * second half of the lane cache key, (source_hash, extraction_version), so it
80
+ * is also the test for "is this preserved row still this lane's current row?".
81
+ * One mapping, used by the in-transaction reattach and by the lanes' cache
82
+ * check (laneDone), so a version bump cannot be applied in one place and missed
83
+ * in the other. `row_kind` is CHECK-constrained to these three kinds.
84
+ */
85
+ const LANE_EXTRACTION_VERSION = {
86
+ fact: typedFacts_1.FACTS_EXTRACTION_VERSION,
87
+ event: eventExtractor_1.EVENTS_EXTRACTION_VERSION,
88
+ preference: preferenceExtractor_1.PREFS_EXTRACTION_VERSION,
89
+ };
90
+ function currentVersion(kind) {
91
+ return LANE_EXTRACTION_VERSION[kind];
92
+ }
93
+ function lane(kind, label, extractor, payloadOf, fallbackText) {
94
+ return {
95
+ kind,
96
+ version: LANE_EXTRACTION_VERSION[kind],
97
+ label,
98
+ async extract(noteTitle, noteBody, chunks) {
99
+ const extracted = await extractor(noteTitle, noteBody);
100
+ return extracted.map((item) => {
101
+ const attributed = (0, writePathAdjudication_1.chunkForCandidate)(chunks, item);
102
+ return {
103
+ payload: payloadOf(item),
104
+ // The chunker gives every note a chunk; without one, 0 makes the row store refuse the row (ProvenanceError).
105
+ sourceChunkId: attributed?.chunkId ?? chunks[0]?.id ?? 0,
106
+ sourceText: attributed?.text ?? fallbackText(item),
107
+ };
108
+ });
109
+ },
110
+ };
111
+ }
112
+ /** The structured-row lanes this pass runs, in order: typed facts (C2), chronicle events (C3), standing preferences (C4). */
113
+ function structuredLanes(options) {
114
+ const lanes = [];
115
+ if (options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)()) {
116
+ lanes.push(lane('fact', 'typed-fact', options.numericFactExtractor, f => ({ value: f.value, unit: f.unit, verb: f.verb, object: f.object, event: f.event }), f => `${f.value} ${f.object}`.trim()));
117
+ }
118
+ if (options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)()) {
119
+ lanes.push(lane('event', 'event', options.eventExtractor, e => ({ when: e.when, event: e.event, entities: e.entities }), e => `${e.when} ${e.event}`.trim()));
120
+ }
121
+ if (options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)()) {
122
+ lanes.push(lane('preference', 'preference', options.preferenceExtractor, p => ({ statement: p.statement, kind: p.kind }), p => p.statement));
123
+ }
124
+ return lanes;
125
+ }
126
+ /**
127
+ * Whether a lane is done for these bytes, so it costs no model call. Either half of the cache key (source_hash,
128
+ * extraction_version) says so: a current row of the lane for this hash, or a completed zero-yield run (#189, lane_runs:
129
+ * a lane that produced nothing has no row to record it).
130
+ */
131
+ function laneDone(db, filePath, lane, noteHash) {
132
+ return !!(0, statements_1.prepareOnce)(db, `SELECT id FROM structured_rows
133
+ WHERE source_path = ? AND row_kind = ? AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, lane.kind, lane.version, noteHash)
134
+ || !!(0, laneRuns_1.laneZeroYieldMarker)(db, filePath, lane.kind, noteHash, lane.version);
135
+ }
136
+ /**
137
+ * Whether the vectors this index holds are in the embedding space the provider writes, so a stored vector may
138
+ * stand in for a fresh one. A new index takes the provider's space; vectors recorded in a different one are
139
+ * never mixed with new ones (that fails the file, before anything changes, until the index is rebuilt); an
140
+ * index that cannot say, or a provider with no identity, reuses nothing.
141
+ */
142
+ function storedVectorsMatch(options) {
143
+ const index = options.vectorIndex;
144
+ const wanted = (0, space_1.embeddingSpace)(options.embeddingProvider);
145
+ if (!wanted || !index.space || !index.setSpace || !index.hasVectors)
146
+ return false;
147
+ const stored = index.space();
148
+ if (stored === wanted)
149
+ return true;
150
+ if (stored !== null) {
151
+ throw new Error(`the index holds vectors from another embedding model (${stored}), not the configured one (${wanted}); `
152
+ + 'rebuild the index rather than mix the two');
153
+ }
154
+ if (index.hasVectors())
155
+ return false; // built before spaces were recorded: it cannot be vouched for
156
+ index.setSpace(wanted);
157
+ return true;
158
+ }
159
+ /** How many chunks, from the top of the note, are stored exactly as the note chunks now: same text, heading and place. */
160
+ function unchangedPrefix(stored, chunks) {
161
+ let n = 0;
162
+ while (n < stored.length && n < chunks.length) {
163
+ const s = stored[n];
164
+ const c = chunks[n];
165
+ if (s.text !== c.text || s.heading_path !== c.headingPath || s.start_line !== c.startLine
166
+ || s.end_line !== c.endLine || s.token_count !== c.tokenCount)
167
+ break;
168
+ n++;
169
+ }
170
+ return n;
171
+ }
172
+ /** The note's row: written before its chunks, as every pass does (the chunks reference it). */
173
+ function upsertNoteRow(db, filePath, row) {
174
+ db.prepare(`
175
+ INSERT INTO notes (path, title, kind, note_hash, modified_at, frontmatter_json)
176
+ VALUES (@path, @title, @kind, @noteHash, @modifiedAt, @frontmatterJson)
177
+ ON CONFLICT(path) DO UPDATE SET
178
+ title = excluded.title,
179
+ kind = excluded.kind,
180
+ note_hash = excluded.note_hash,
181
+ modified_at = excluded.modified_at,
182
+ frontmatter_json = excluded.frontmatter_json
183
+ `).run({ path: filePath, ...row });
184
+ }
185
+ /** SQLite's LIKE and COLLATE NOCASE fold the ASCII letters and nothing else; so does this. */
186
+ const asciiLower = (s) => s.replace(/[A-Z]/g, c => String.fromCharCode(c.charCodeAt(0) + 32));
187
+ /**
188
+ * The entity each wikilink points at, keyed by the link with its ASCII letters lowercased. A link names the entity whose
189
+ * note's path ends in `/<link>.md` or whose name is the link (ASCII case-insensitive), and where several do, the one with
190
+ * the lowest id. That is what a query per link with `path LIKE '%/<link>.md' OR name = ?` answered, but a leading wildcard
191
+ * cannot use an index, so that cost a scan of every note per distinct link: a 61 KB note of 7,000 links took 22 s in a
192
+ * 2,000-note vault. A link nothing names has no entry. The claims are kept (see claimsOf), so a note does not scan them.
193
+ */
194
+ function resolveWikilinks(db, links) {
195
+ const wanted = new Set(links.map(asciiLower));
196
+ const found = new Map();
197
+ if (wanted.size === 0)
198
+ return found;
199
+ const byKey = claimsOf(db);
200
+ for (const key of wanted) {
201
+ const ids = byKey.get(key);
202
+ if (ids !== undefined)
203
+ found.set(key, ids[0]);
204
+ }
205
+ return found;
206
+ }
207
+ /** The keys an entity answers to: its name, and each path below its note's folder (see resolveWikilinks). */
208
+ function claimKeysOf(name, path) {
209
+ const keys = [asciiLower(name)];
210
+ const lowerPath = asciiLower(path);
211
+ if (lowerPath.endsWith('.md')) {
212
+ const stem = lowerPath.slice(0, -3);
213
+ for (let at = stem.indexOf('/'); at !== -1; at = stem.indexOf('/', at + 1))
214
+ keys.push(stem.slice(at + 1));
215
+ }
216
+ return keys;
217
+ }
218
+ const claimsByDb = new WeakMap();
219
+ function entityFingerprint(db) {
220
+ return (0, statements_1.prepareOnce)(db, "SELECT COALESCE(MAX(id), 0) || '|' || COUNT(*) FROM entities").pluck().get();
221
+ }
222
+ function addClaim(byKey, key, id) {
223
+ const ids = byKey.get(key);
224
+ // Ids arrive in ascending order (a new entity is the newest), so the last one is the only one this id can repeat.
225
+ if (ids === undefined)
226
+ byKey.set(key, [id]);
227
+ else if (ids[ids.length - 1] !== id)
228
+ ids.push(id);
229
+ }
230
+ function dropClaim(byKey, key, id) {
231
+ const ids = byKey.get(key);
232
+ if (ids === undefined)
233
+ return;
234
+ const at = ids.indexOf(id);
235
+ if (at !== -1)
236
+ ids.splice(at, 1);
237
+ if (ids.length === 0)
238
+ byKey.delete(key);
239
+ }
240
+ function claimsOf(db) {
241
+ const fingerprint = entityFingerprint(db);
242
+ const held = claimsByDb.get(db);
243
+ if (held !== undefined && held.fingerprint === fingerprint)
244
+ return held.byKey;
245
+ const byKey = new Map();
246
+ const rows = db.prepare('SELECT e.id AS id, e.canonical_name AS name, n.path AS path FROM entities e JOIN notes n ON e.source_note = n.path ORDER BY e.id').iterate();
247
+ for (const row of rows) {
248
+ for (const key of claimKeysOf(row.name, row.path))
249
+ addClaim(byKey, key, row.id);
250
+ }
251
+ claimsByDb.set(db, { fingerprint, byKey });
252
+ return byKey;
253
+ }
254
+ /** The entity rows of one note, as the claims read them (joined to their note, as resolveWikilinks joins them). */
255
+ function entityRowsOf(db, filePath) {
256
+ return (0, statements_1.prepareOnce)(db, 'SELECT e.id AS id, e.canonical_name AS name, n.path AS path FROM entities e JOIN notes n ON e.source_note = n.path WHERE e.source_note = ?')
257
+ .all(filePath);
258
+ }
259
+ /**
260
+ * Follows one note's entity change (rebuildNoteRows: its old entity deleted, its new one inserted): what the old rows claimed
261
+ * is dropped and what the new rows claim is added, if the claims were current before the change. If they were not, they are
262
+ * dropped and read again on the next resolve.
263
+ */
264
+ function followEntityChange(db, fingerprintBefore, removed, added) {
265
+ const held = claimsByDb.get(db);
266
+ if (held === undefined)
267
+ return;
268
+ if (held.fingerprint !== fingerprintBefore) {
269
+ claimsByDb.delete(db);
270
+ return;
271
+ }
272
+ for (const row of removed)
273
+ for (const key of claimKeysOf(row.name, row.path))
274
+ dropClaim(held.byKey, key, row.id);
275
+ for (const row of added)
276
+ for (const key of claimKeysOf(row.name, row.path))
277
+ addClaim(held.byKey, key, row.id);
278
+ held.fingerprint = entityFingerprint(db);
279
+ }
280
+ /** A note's entity, frontmatter facts and wikilink relations, written again from its parse. Returns the entity's id. */
281
+ function rebuildNoteRows(db, filePath, note, kind, engineWrote, now) {
282
+ // Frontmatter/LLM note facts use this index instant (and the default global scope). Keep the workspace's
283
+ // observation even when a later pass replaces the rows after a backward clock step, or a note is retired.
284
+ // This shares the note-row transaction, so a rolled-back pass cannot advance the read clock.
285
+ db.prepare(`INSERT INTO note_index_clock (scope, observed_at) VALUES ('global', ?)
286
+ ON CONFLICT(scope) DO UPDATE SET observed_at = MAX(note_index_clock.observed_at, excluded.observed_at)`)
287
+ .run(now);
288
+ // 10. Extract entities — delete old, insert new (the wikilink claims follow the change, see followEntityChange)
289
+ const fingerprintBefore = entityFingerprint(db);
290
+ const removedEntities = entityRowsOf(db, filePath);
291
+ db.prepare('DELETE FROM entities WHERE source_note = ?').run(filePath);
292
+ const rawAliases = note.frontmatter.aliases;
293
+ const aliases = Array.isArray(rawAliases)
294
+ ? rawAliases.filter((a) => typeof a === 'string')
295
+ : typeof rawAliases === 'string'
296
+ ? [rawAliases]
297
+ : [];
298
+ if (rawAliases !== undefined && rawAliases !== null
299
+ && (Array.isArray(rawAliases) ? aliases.length < rawAliases.length : typeof rawAliases !== 'string')) {
300
+ warnFrontmatter(filePath, 'front matter "aliases" has a value that is not text');
301
+ }
302
+ const entityRow = db.prepare(`
303
+ INSERT INTO entities (type, canonical_name, aliases_json, source_note, confidence, updated_at)
304
+ VALUES (@type, @canonicalName, @aliasesJson, @sourceNote, @confidence, @updatedAt)
305
+ RETURNING id
306
+ `).get({
307
+ type: kind ?? 'note',
308
+ canonicalName: note.title,
309
+ aliasesJson: JSON.stringify(aliases),
310
+ sourceNote: filePath,
311
+ confidence: 1.0,
312
+ updatedAt: now,
313
+ });
314
+ followEntityChange(db, fingerprintBefore, removedEntities, entityRowsOf(db, filePath));
315
+ // 11. Extract facts — delete old, insert new
316
+ db.prepare('DELETE FROM facts WHERE source_path = ?').run(filePath);
317
+ const insertFact = db.prepare(`
318
+ INSERT INTO facts (subject_entity_id, predicate, object_text, source_path, confidence, valid_from, updated_at)
319
+ VALUES (@subjectEntityId, @predicate, @objectText, @sourcePath, @confidence, @validFrom, @updatedAt)
320
+ `);
321
+ // The engine's own header in a daily note is bookkeeping, not something anyone said: the day the file is about, which
322
+ // its name already says. Only a note the engine wrote has it (`engineWrote`, told from its front matter alone, not
323
+ // its place, its name or its body), and only that value is dropped, never one a person put there (dailyNote.ts).
324
+ for (const predicate of FACT_PREDICATES) {
325
+ const value = note.frontmatter[predicate];
326
+ if (value === undefined || value === null)
327
+ continue;
328
+ if (predicate === dailyNote_1.DAILY_NOTE_HEADER_KEY && engineWrote && (0, dailyNote_1.isEngineHeaderDate)(filePath, value))
329
+ continue;
330
+ const objectText = Array.isArray(value) ? value.map(v => (v == null ? '' : factText(v))).join(', ') : factText(value);
331
+ insertFact.run({
332
+ subjectEntityId: entityRow.id,
333
+ predicate,
334
+ objectText,
335
+ sourcePath: filePath,
336
+ confidence: 1.0,
337
+ validFrom: now,
338
+ updatedAt: now,
339
+ });
340
+ }
341
+ // 12. Save wikilink relations — delete old, insert new
342
+ db.prepare('DELETE FROM relations WHERE source_note = ?').run(filePath);
343
+ const insertRelation = db.prepare(`
344
+ INSERT INTO relations (source_entity_id, relation, target_entity_id, source_note, confidence)
345
+ VALUES (?, 'links_to', ?, ?, 1.0)
346
+ `);
347
+ const targets = resolveWikilinks(db, note.wikilinks);
348
+ for (const wikilink of note.wikilinks) {
349
+ const target = targets.get(asciiLower(wikilink));
350
+ if (target !== undefined)
351
+ insertRelation.run(entityRow.id, target, filePath);
352
+ }
353
+ return entityRow.id;
354
+ }
355
+ /**
356
+ * After a pass's rows are committed: the written chunks take the vectors of replaced chunks with the same text, the
357
+ * replaced chunks' vectors go, the rest are embedded, and only then is the note's hash committed (two-phase).
358
+ */
359
+ async function storeVectors(db, filePath, options, pass) {
360
+ // A chunk whose text did not change keeps the vector it already has: copy it to the replacement
361
+ // chunk before the old vectors go, and embed only the chunks with new text. An edited or
362
+ // appended file then costs what changed, not what it holds (a ledger is one chunk per event).
363
+ const { insertedChunks, kept, oldChunkIds, vectorSource, vectorsMatch, embedsHere, noteHash } = pass;
364
+ const reused = new Set();
365
+ if (vectorsMatch && options.vectorIndex?.copy) {
366
+ for (const [chunkId, fromId] of vectorSource) {
367
+ if (options.vectorIndex.copy(fromId, chunkId))
368
+ reused.add(chunkId);
369
+ }
370
+ }
371
+ if (options.vectorIndex) {
372
+ for (const id of oldChunkIds) {
373
+ options.vectorIndex.delete(id);
374
+ }
375
+ }
376
+ // 9. Embed chunks if provider given (outside transaction — async)
377
+ if (embedsHere && options.embeddingProvider && options.vectorIndex) {
378
+ const { embeddingProvider, vectorIndex } = options;
379
+ const toEmbed = insertedChunks.slice(kept).filter(c => !reused.has(c.id));
380
+ const texts = toEmbed.map(c => c.text);
381
+ if (texts.length > 0) {
382
+ const embeddings = await embeddingProvider.embed(texts, options.embedBudgetMs === undefined ? undefined : { budgetMs: options.embedBudgetMs * Math.ceil(texts.length / EMBED_REQUEST_TEXTS) });
383
+ // A vector missing from the answer fails the file: its hash stays '' (see transactionHash), so the next pass
384
+ // embeds the note again. Committing the hash here would leave those chunks without a vector for good, since an
385
+ // unchanged note is never read again.
386
+ if (embeddings.length !== texts.length || texts.some((_, i) => !embeddings[i])) {
387
+ throw new Error(`embedding service answered ${embeddings.length} vectors for ${texts.length} chunks of ${filePath}`);
388
+ }
389
+ for (let i = 0; i < toEmbed.length; i++) {
390
+ if (vectorIndex.insertNew)
391
+ vectorIndex.insertNew(toEmbed[i].id, embeddings[i]);
392
+ else
393
+ vectorIndex.upsert(toEmbed[i].id, embeddings[i]);
394
+ }
395
+ }
396
+ // Fix #2 (cont): Commit the real note_hash only after embedding succeeds
397
+ db.prepare('UPDATE notes SET note_hash = ? WHERE path = ?').run(noteHash, filePath);
398
+ }
399
+ }
400
+ /** Per database: the last pass's AppendBase of each file, for at most this many files, the most recently indexed. */
401
+ const MAX_APPEND_BASES = 64;
402
+ const appendBases = new WeakMap();
403
+ /** The file's base, removed: a pass that fails leaves none, and one that succeeds records its own. */
404
+ function takeAppendBase(db, filePath) {
405
+ const bases = appendBases.get(db);
406
+ const base = bases?.get(filePath);
407
+ bases?.delete(filePath);
408
+ return base;
409
+ }
410
+ function recordAppendBase(db, filePath, base) {
411
+ let bases = appendBases.get(db);
412
+ if (!bases)
413
+ appendBases.set(db, bases = new Map());
414
+ bases.delete(filePath);
415
+ bases.set(filePath, base);
416
+ if (bases.size > MAX_APPEND_BASES)
417
+ bases.delete(bases.keys().next().value);
418
+ }
419
+ const chunkingOf = (options) => `${options.chunkOptions?.maxTokens ?? 400}/${options.chunkOptions?.overlapTokens ?? 0}`;
420
+ const hashKey = (chunkHash) => parseInt(chunkHash.slice(0, 12), 16);
421
+ /** Whether any extraction lane would read the note: each reads all of it, so such a pass is a full one. */
422
+ function laneReadsNote(options) {
423
+ return !!options.factExtractor
424
+ || (0, factsWritePath_1.factsWritePathEnabled)()
425
+ || !!(options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)())
426
+ || !!(options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)())
427
+ || !!(options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)());
428
+ }
429
+ /** The base a full pass leaves, or null when an append to the note could not be indexed by its appended part. */
430
+ function appendBaseOf(filePath, raw, content, detailed, kind, cut, rows, noteHash, options) {
431
+ if ((0, dailyNote_1.isDailyNotePath)(filePath) || raw.length === 0 || raw[raw.length - 1] !== 0x0a || !(0, buffer_1.isUtf8)(raw))
432
+ return null;
433
+ if (!detailed.titleFromHeading || !cut.last || cut.lastFirst === undefined)
434
+ return null;
435
+ if (cut.lastFirst >= rows.ids.length)
436
+ return null;
437
+ if (!(0, parser_1.wikilinksSettled)(detailed.rawBody))
438
+ return null;
439
+ // Where the last section's first line starts in the file: the body is the text after the frontmatter, trimmed.
440
+ const lines = detailed.note.body.split('\n');
441
+ let at = content.length - detailed.rawBody.trimStart().length;
442
+ for (let i = 0; i < cut.last.line; i++)
443
+ at += lines[i].length + 1;
444
+ if (!content.startsWith(lines[cut.last.line], at))
445
+ return null;
446
+ return {
447
+ bytes: raw.length,
448
+ noteHash,
449
+ chunking: chunkingOf(options),
450
+ title: detailed.note.title,
451
+ kind,
452
+ frontmatter: detailed.note.frontmatter,
453
+ frontmatterJson: JSON.stringify(detailed.note.frontmatter),
454
+ wikilinks: detailed.note.wikilinks,
455
+ last: { ...cut.last, byte: Buffer.byteLength(content.slice(0, at)) },
456
+ tailFirstId: rows.ids[cut.lastFirst],
457
+ headCount: cut.lastFirst,
458
+ headHashes: new Set(rows.hashes.slice(0, cut.lastFirst).map(hashKey)),
459
+ };
460
+ }
461
+ /**
462
+ * Index `raw`, the file's bytes, by what was appended to the bytes `base` describes (see AppendBase). Returns false,
463
+ * having changed nothing, when it cannot; the caller then indexes the file in full.
464
+ */
465
+ async function indexAppend(db, filePath, raw, options, base) {
466
+ if (raw.length <= base.bytes || chunkingOf(options) !== base.chunking || laneReadsNote(options))
467
+ return false;
468
+ // The bytes that were indexed, unchanged, then valid text: hashed on the way to the note's new hash.
469
+ const hash = (0, crypto_1.createHash)('sha256').update(raw.subarray(0, base.bytes));
470
+ if (hash.copy().digest('hex') !== base.noteHash)
471
+ return false;
472
+ const appended = raw.subarray(base.bytes);
473
+ if (!(0, buffer_1.isUtf8)(appended))
474
+ return false;
475
+ const noteHash = hash.update(appended).digest('hex');
476
+ // The note from its last section on: that section as it was indexed, then what was appended. The body is trimmed,
477
+ // and this text starts with a line that is not blank (a heading, or the body's first line), so trimming its end
478
+ // trims the body's.
479
+ const kept = raw.toString('utf8', base.last.byte, base.bytes);
480
+ const tail = kept + appended.toString('utf8');
481
+ const lines = tail.trimEnd().split('\n');
482
+ const cut = (0, chunker_1.chunkBodyFrom)(filePath, lines, base.last, options.chunkOptions);
483
+ if (!cut.last || cut.lastFirst === undefined || cut.lastFirst >= cut.chunks.length)
484
+ return false;
485
+ const wikilinks = [...new Set([...base.wikilinks, ...(0, parser_1.wikilinkTargets)(tail, kept.length)])];
486
+ const stat = await promises_1.default.stat(filePath);
487
+ const modifiedAt = stat.mtime.toISOString();
488
+ const now = new Date().toISOString();
489
+ const embeddingConfigured = !!(options.embeddingProvider && options.vectorIndex);
490
+ const vectorsMatch = embeddingConfigured && storedVectorsMatch(options);
491
+ const embedsHere = embeddingConfigured && !(vectorsMatch && options.vectorIndex.rebuilding?.() === true);
492
+ const write = db.transaction(() => {
493
+ const existing = db.prepare('SELECT note_hash FROM notes WHERE path = ?').pluck().get(filePath);
494
+ if (existing !== base.noteHash)
495
+ return null;
496
+ if (db.prepare('SELECT 1 FROM structured_rows WHERE source_path = ? LIMIT 1').get(filePath))
497
+ return null;
498
+ const oldTail = db.prepare(`SELECT id, heading_path, text, start_line, end_line, token_count, chunk_hash FROM chunks
499
+ WHERE note_path = ? AND id >= ? ORDER BY id`).all(filePath, base.tailFirstId);
500
+ const headIds = db.prepare('SELECT id FROM chunks WHERE note_path = ? AND id < ? ORDER BY id')
501
+ .pluck().all(filePath, base.tailFirstId);
502
+ if (headIds.length !== base.headCount)
503
+ return null;
504
+ // As a full pass does: the chunks from the top that are stored as they are now are kept (all of those before the
505
+ // last section, whose lines did not change), and only if each kept chunk has its vector when the pass embeds.
506
+ const keep = unchangedPrefix(oldTail, cut.chunks);
507
+ const keptIds = [...headIds, ...oldTail.slice(0, keep).map(c => c.id)];
508
+ if (keptIds.length > 0 && embedsHere
509
+ && (!vectorsMatch || options.vectorIndex.countStored?.(keptIds) !== keptIds.length))
510
+ return null;
511
+ const added = cut.chunks.slice(keep).map(chunk => ({ chunk, chunkHash: (0, hasher_1.hashContent)(chunk.text) }));
512
+ if (added.some(a => base.headHashes.has(hashKey(a.chunkHash))))
513
+ return null;
514
+ upsertNoteRow(db, filePath, {
515
+ title: base.title, kind: base.kind, noteHash: embedsHere ? '' : noteHash, modifiedAt, frontmatterJson: base.frontmatterJson,
516
+ });
517
+ if (keep < oldTail.length)
518
+ db.prepare('DELETE FROM chunks WHERE note_path = ? AND id >= ?').run(filePath, oldTail[keep].id);
519
+ const oldIdByHash = new Map();
520
+ for (const old of oldTail)
521
+ if (!oldIdByHash.has(old.chunk_hash))
522
+ oldIdByHash.set(old.chunk_hash, old.id);
523
+ const insertChunk = db.prepare(`
524
+ INSERT INTO chunks (note_path, heading_path, text, start_line, end_line, token_count, chunk_hash)
525
+ VALUES (@notePath, @headingPath, @text, @startLine, @endLine, @tokenCount, @chunkHash)
526
+ RETURNING id
527
+ `);
528
+ const insertedChunks = oldTail.slice(0, keep).map((old, i) => ({ id: old.id, text: cut.chunks[i].text }));
529
+ const vectorSource = new Map();
530
+ for (const { chunk, chunkHash } of added) {
531
+ const row = insertChunk.get({
532
+ notePath: chunk.notePath,
533
+ headingPath: chunk.headingPath,
534
+ text: chunk.text,
535
+ startLine: chunk.startLine,
536
+ endLine: chunk.endLine,
537
+ tokenCount: chunk.tokenCount,
538
+ chunkHash,
539
+ });
540
+ insertedChunks.push({ id: row.id, text: chunk.text });
541
+ const sameText = oldIdByHash.get(chunkHash);
542
+ if (sameText !== undefined)
543
+ vectorSource.set(row.id, sameText);
544
+ }
545
+ rebuildNoteRows(db, filePath, { title: base.title, frontmatter: base.frontmatter, wikilinks }, base.kind, false, now);
546
+ return {
547
+ insertedChunks,
548
+ kept: keep,
549
+ oldChunkIds: oldTail.slice(keep).map(old => old.id),
550
+ vectorSource,
551
+ chunkHashes: [...oldTail.slice(0, keep).map(old => old.chunk_hash), ...added.map(a => a.chunkHash)],
552
+ };
553
+ });
554
+ const written = write();
555
+ if (!written)
556
+ return false;
557
+ await storeVectors(db, filePath, options, { ...written, vectorsMatch, embedsHere, noteHash });
558
+ // The next append starts from this note's new last section.
559
+ if (raw[raw.length - 1] === 0x0a && (0, parser_1.wikilinksSettled)(tail, kept.length)) {
560
+ let at = 0;
561
+ for (let i = base.last.line; i < cut.last.line; i++)
562
+ at += lines[i - base.last.line].length + 1;
563
+ for (const chunkHash of written.chunkHashes.slice(0, cut.lastFirst))
564
+ base.headHashes.add(hashKey(chunkHash));
565
+ recordAppendBase(db, filePath, {
566
+ ...base,
567
+ bytes: raw.length,
568
+ noteHash,
569
+ wikilinks,
570
+ last: { ...cut.last, byte: base.last.byte + Buffer.byteLength(tail.slice(0, at)) },
571
+ tailFirstId: written.insertedChunks[cut.lastFirst].id,
572
+ headCount: base.headCount + cut.lastFirst,
573
+ });
574
+ }
575
+ return true;
576
+ }
577
+ async function readRegularFile(filePath) {
578
+ try {
579
+ return await promises_1.default.readFile(filePath, { flag: fs_1.constants.O_RDONLY | fs_1.constants.O_NOFOLLOW | fs_1.constants.O_NONBLOCK });
580
+ }
581
+ catch (err) {
582
+ if (err.code === 'ELOOP')
583
+ throw new Error(`refusing to index ${JSON.stringify(filePath)}: it is a symbolic link`);
584
+ throw err;
585
+ }
586
+ }
587
+ /**
588
+ * Index one file. Runs under the same per-file lock every writer of that file takes, so an index
589
+ * pass can never read a file, lose the race to a rewrite (memory retirement), and then commit the
590
+ * stale text over the newer index.
591
+ */
592
+ function indexFile(db, filePath, options = {}) {
593
+ return (0, writer_1.withDailyFileLock)(filePath, () => indexFileUnlocked(db, filePath, options));
594
+ }
595
+ async function indexFileUnlocked(db, filePath, options = {}) {
596
+ // 0. A note over the cap is not read (a read allocates the file, and parsing it a multiple of that)
597
+ const { size } = await promises_1.default.stat(filePath);
598
+ if (size > exports.MAX_NOTE_BYTES)
599
+ throw new NoteTooLargeError(filePath, size);
600
+ // 1. Read file content. Never through a symbolic link: the walker does not list one, and a caller that names one (the
601
+ // watcher is told of a link to a file as the file) must not pull a file from outside the vault into recall.
602
+ const raw = await readRegularFile(filePath);
603
+ // A note this process indexed that has only been appended to since is indexed by what was appended (see
604
+ // indexAppend); anything else is read and indexed in full, below.
605
+ const base = takeAppendBase(db, filePath);
606
+ if (base && await indexAppend(db, filePath, raw, options, base))
607
+ return;
608
+ const content = raw.toString('utf8');
609
+ // 2. Hash content (a daily note's hash also names the layout its chunks are cut in, see dailyEntries.ts)
610
+ const noteHash = (0, dailyEntries_1.noteContentHash)(filePath, content);
611
+ // 3. Check if unchanged
612
+ const existing = db.prepare('SELECT note_hash FROM notes WHERE path = ?').get(filePath);
613
+ const noteUnchanged = !!(existing && existing.note_hash === noteHash);
614
+ if (noteUnchanged && base?.noteHash === noteHash)
615
+ recordAppendBase(db, filePath, base); // it still describes the note
616
+ if (noteUnchanged) {
617
+ // The facts of an unchanged note are not rebuilt below, so a row an earlier build made of the engine's own header
618
+ // would stay. Remove it here: one statement for a daily note, no chunk or extractor involved.
619
+ removeDailyHeaderFacts(db, filePath, content);
620
+ // C2: unchanged content skips extraction ONLY when every armed lane is done for these bytes (ADR §4.2: the
621
+ // cache key is (source_hash, extraction_version), see laneDone). A lane without rows (first run, version bump)
622
+ // falls through to re-extract; with no lane armed the note is simply skipped.
623
+ if (structuredLanes(options).every(lane => laneDone(db, filePath, lane, noteHash)))
624
+ return;
625
+ }
626
+ // 4. Parse note
627
+ const detailed = (0, parser_1.parseNoteDetailed)(filePath, content);
628
+ const parsed = detailed.note;
629
+ // 5. Get mtime
630
+ const stat = await promises_1.default.stat(filePath);
631
+ const modifiedAt = stat.mtime.toISOString();
632
+ const now = new Date().toISOString();
633
+ const kind = noteKind(filePath, parsed.frontmatter);
634
+ // HARMONIZATION: lane ownership — when the fleet's typed-facts lane is armed
635
+ // (extractor + RENOR_TYPED_FACTS), it is THE fact-lane candidate generator;
636
+ // this branch's local-rules capture lane (RENOR_FACTS_WRITE_PATH) stands
637
+ // down for that note so the same note is never fact-extracted twice (two
638
+ // generators would double-write the same identities under two extraction
639
+ // versions). With only RENOR_FACTS_WRITE_PATH on, behavior is unchanged
640
+ // from this branch (local rules + DecisionModel seam).
641
+ const typedFactsLaneArmed = !!(options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)());
642
+ const embeddingConfigured = !!(options.embeddingProvider && options.vectorIndex);
643
+ // Stored vectors may be kept or copied only if they are in the space this provider writes.
644
+ const vectorsMatch = embeddingConfigured && storedVectorsMatch(options);
645
+ // While the index is unfinished in this provider's space, the background job that completes it embeds every chunk
646
+ // that has no vector, this file's included. The file's chunks are then written without vectors and left to it, so
647
+ // no chunk is embedded twice and a save embeds nothing it did not change (a ledger appended to keeps its earlier
648
+ // events as they are). Decided in the same synchronous step as the write below, so the job cannot have finished
649
+ // in between: it ends only when no chunk lacks a vector.
650
+ const embedsHere = embeddingConfigured && !(vectorsMatch && options.vectorIndex.rebuilding?.() === true);
651
+ // Fix #2: Use two-phase note_hash commit. If this file embeds its chunks, store ''
652
+ // so that a failed embed() causes the file to be re-processed on the next run.
653
+ // Store the real hash only after embeddings succeed (or immediately if nothing is embedded here).
654
+ const transactionHash = embedsHere ? '' : noteHash;
655
+ // 6-12. Wrap all DB mutations in a transaction.
656
+ // Fix #1: Query old chunk IDs inside the transaction so they are only deleted
657
+ // from the vector index AFTER the transaction commits successfully.
658
+ const upsertAll = db.transaction(() => {
659
+ // The stored chunks, top to bottom (a note's chunk ids ascend with position). The ones that
660
+ // are replaced have their vectors deleted after commit.
661
+ const oldChunks = db.prepare('SELECT id, heading_path, text, start_line, end_line, token_count, chunk_hash FROM chunks WHERE note_path = ? ORDER BY id').all(filePath);
662
+ // The old chunk holding each text: a chunk whose text is unchanged keeps its vector.
663
+ const oldIdByHash = new Map();
664
+ for (const old of oldChunks)
665
+ if (!oldIdByHash.has(old.chunk_hash))
666
+ oldIdByHash.set(old.chunk_hash, old.id);
667
+ const cut = (0, chunker_1.chunkNoteCut)(parsed, options.chunkOptions);
668
+ const chunks = cut.chunks;
669
+ // 6. Upsert note row
670
+ upsertNoteRow(db, filePath, {
671
+ title: parsed.title, kind, noteHash: transactionHash, modifiedAt, frontmatterJson: JSON.stringify(parsed.frontmatter),
672
+ });
673
+ // 7. Delete old chunks. structured_rows.source_chunk_id cascades, and
674
+ // the per-lane cache check happens after this rewrite. Copy current rows
675
+ // out first and reattach them to the replacement chunk, or a current lane
676
+ // looks stale and is extracted again.
677
+ const preservedStructured = db.prepare(`SELECT r.row_kind, r.source_hash, r.scope, r.extraction_version, r.valid_from, r.valid_to,
678
+ r.payload_json, r.dedupe_key, r.confirm_count, r.created_at, c.text AS source_text
679
+ FROM structured_rows r
680
+ LEFT JOIN chunks c ON c.id = r.source_chunk_id
681
+ WHERE r.source_path = ?`).all(filePath);
682
+ // The chunks from the top of an edited note that are already stored exactly as they are now
683
+ // keep their rows, search entries and vectors; only the rest is replaced (a ledger appended to
684
+ // replaces its last event and adds the new one). The whole note is replaced instead when the
685
+ // bytes are unchanged (a lane re-run), when the last pass over it did not finish (an empty
686
+ // hash), when structured rows hang off its chunks, or when a stored chunk lacks its vector,
687
+ // so a full pass heals it, as it always did.
688
+ let keep = 0;
689
+ if (existing && !noteUnchanged && existing.note_hash !== '' && preservedStructured.length === 0) {
690
+ keep = unchangedPrefix(oldChunks, chunks);
691
+ if (keep > 0 && embedsHere
692
+ && (!vectorsMatch || options.vectorIndex.countStored?.(oldChunks.slice(0, keep).map(c => c.id)) !== keep)) {
693
+ keep = 0;
694
+ }
695
+ }
696
+ if (keep === 0)
697
+ db.prepare('DELETE FROM chunks WHERE note_path = ?').run(filePath);
698
+ else
699
+ db.prepare('DELETE FROM chunks WHERE note_path = ? AND id > ?').run(filePath, oldChunks[keep - 1].id);
700
+ // 8. Insert the chunks that are not kept, collecting inserted IDs
701
+ const insertChunk = db.prepare(`
702
+ INSERT INTO chunks (note_path, heading_path, text, start_line, end_line, token_count, chunk_hash)
703
+ VALUES (@notePath, @headingPath, @text, @startLine, @endLine, @tokenCount, @chunkHash)
704
+ RETURNING id
705
+ `);
706
+ const insertedChunks = oldChunks.slice(0, keep).map((old, i) => ({ id: old.id, text: chunks[i].text }));
707
+ const chunkHashes = oldChunks.slice(0, keep).map(old => old.chunk_hash); // of every chunk, in order
708
+ const vectorSource = new Map(); // new chunk id -> old chunk id with the same text
709
+ for (const chunk of chunks.slice(keep)) {
710
+ const chunkHash = (0, hasher_1.hashContent)(chunk.text);
711
+ chunkHashes.push(chunkHash);
712
+ const row = insertChunk.get({
713
+ notePath: chunk.notePath,
714
+ headingPath: chunk.headingPath,
715
+ text: chunk.text,
716
+ startLine: chunk.startLine,
717
+ endLine: chunk.endLine,
718
+ tokenCount: chunk.tokenCount,
719
+ chunkHash,
720
+ });
721
+ insertedChunks.push({ id: row.id, text: chunk.text });
722
+ const sameText = oldIdByHash.get(chunkHash);
723
+ if (sameText !== undefined)
724
+ vectorSource.set(row.id, sameText);
725
+ }
726
+ // Reattach rows copied out before the chunk delete. A lane whose version
727
+ // and hash are still current keeps its row; a stale lane stays deleted so
728
+ // only that lane re-extracts. The row keeps the replacement chunk that
729
+ // still contains its source text. Chunk 0 is only the fallback when that
730
+ // text is gone; it is not the provenance of every preserved row.
731
+ if (preservedStructured.length > 0 && insertedChunks.length > 0) {
732
+ const reattach = db.prepare(`INSERT INTO structured_rows
733
+ (row_kind, source_path, source_hash, source_chunk_id, scope, extraction_version,
734
+ valid_from, valid_to, payload_json, dedupe_key, confirm_count, created_at)
735
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
736
+ for (const saved of preservedStructured) {
737
+ const kind = saved.row_kind;
738
+ const version = saved.extraction_version;
739
+ const hash = saved.source_hash;
740
+ const current = saved.valid_to == null
741
+ && version === currentVersion(kind)
742
+ && hash === noteHash;
743
+ if (!current)
744
+ continue;
745
+ const sourceText = typeof saved.source_text === 'string' ? saved.source_text : '';
746
+ const attributed = sourceText
747
+ ? (0, writePathAdjudication_1.chunkForCandidate)(insertedChunks, { statement: sourceText })
748
+ : null;
749
+ const chunkId = attributed?.chunkId ?? insertedChunks[0].id;
750
+ reattach.run(kind, filePath, hash, chunkId, saved.scope, version, saved.valid_from, saved.valid_to, saved.payload_json, saved.dedupe_key, saved.confirm_count, saved.created_at);
751
+ }
752
+ }
753
+ // 10-12. Entities, frontmatter facts and wikilink relations: delete old, insert new
754
+ const entityId = rebuildNoteRows(db, filePath, parsed, kind, (0, dailyNote_1.isEngineDailyNote)(filePath, content), now);
755
+ return {
756
+ insertedChunks,
757
+ kept: keep,
758
+ oldChunkIds: oldChunks.slice(keep).map(old => old.id),
759
+ vectorSource,
760
+ entityId,
761
+ cut,
762
+ chunkHashes,
763
+ };
764
+ });
765
+ // Fix #1 (cont): Delete old vectors only after transaction commits successfully
766
+ const { insertedChunks, kept, oldChunkIds, vectorSource, entityId, cut, chunkHashes } = upsertAll();
767
+ await storeVectors(db, filePath, options, { insertedChunks, kept, oldChunkIds, vectorSource, vectorsMatch, embedsHere, noteHash });
768
+ // 11b. LLM fact extraction — runs outside transaction, after embeddings (async/network).
769
+ // Fix #5: On extraction failure, reset note_hash to '' so the next unchanged-file
770
+ // indexing pass re-processes this note and retries extraction. Otherwise the real
771
+ // hash committed above (embedding path) or in the transaction marks the note as
772
+ // fully current while its LLM facts are missing/stale — and the transaction has
773
+ // already deleted the prior facts for this source, making the loss sticky.
774
+ if (options.factExtractor) {
775
+ try {
776
+ const llmFacts = await options.factExtractor(parsed.title, parsed.body);
777
+ if (llmFacts.length > 0) {
778
+ const insertLlmFact = db.prepare(`
779
+ INSERT INTO facts (subject_entity_id, predicate, object_text, source_path, confidence, valid_from, updated_at)
780
+ VALUES (?, ?, ?, ?, ?, ?, ?)
781
+ `);
782
+ for (const fact of llmFacts) {
783
+ insertLlmFact.run(entityId, fact.predicate, `${fact.subject}: ${fact.object}`, filePath, fact.confidence, now, now);
784
+ }
785
+ }
786
+ }
787
+ catch (err) {
788
+ log.error('extraction_failed', { lane: 'LLM', note: path_1.default.basename(filePath) }, err);
789
+ // #188: reset the hash so the next pass retries (Fix #5), AND rethrow.
790
+ // Swallowing it here left `indexVault` counting the note as INDEXED, so
791
+ // `index` reported 0 errors and exited 0 with this note's facts missing —
792
+ // the blanket catch #188 is about. The structured lanes below already
793
+ // rethrow; this lane now fails the same way, so a real failure reaches the
794
+ // process exit status. Every other note still indexes.
795
+ db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
796
+ throw err;
797
+ }
798
+ }
799
+ // C2 facts write-path (flag-gated, default OFF): save-time typed fact
800
+ // capture into the structured-row store through the real index pipeline.
801
+ // Flag off = zero decision calls, zero new writes, byte-identical shipped
802
+ // behavior. The bitemporal model is the rowStore's identity-keyed
803
+ // supersession (the markdown ledger stays `_mcp_`-lifecycle-only per
804
+ // inventory-facts). Fix #5 pattern: a capture failure resets note_hash so
805
+ // the next pass retries — failed capture never marks a note current.
806
+ // HARMONIZATION: stands down when the fleet's typed-facts lane is armed
807
+ // (single fact-lane candidate generator — see lane ownership above).
808
+ if (!typedFactsLaneArmed && (0, factsWritePath_1.factsWritePathEnabled)()) {
809
+ try {
810
+ await (0, factsWritePath_1.captureNoteFacts)(db, {
811
+ notePath: filePath,
812
+ noteTitle: parsed.title,
813
+ frontmatter: parsed.frontmatter,
814
+ sourceHash: noteHash,
815
+ chunks: insertedChunks,
816
+ }, options.decisionModel);
817
+ }
818
+ catch (err) {
819
+ log.error('extraction_failed', { lane: 'facts write-path capture', note: path_1.default.basename(filePath) }, err);
820
+ // Same #188 contract as the legacy lane above: reset the hash so the next
821
+ // pass retries, then rethrow so the failure is counted and reaches the
822
+ // exit status instead of being hidden by a blanket catch.
823
+ db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
824
+ throw err;
825
+ }
826
+ }
827
+ // 11c-e. The structured-row lanes (typed facts, chronicle events, standing preferences; each flag-gated, default
828
+ // off; ADR-0003 §4). A lane already done for these bytes (see laneDone) costs no model call. Otherwise its
829
+ // previous rows stay until the extractor returns, so an extractor that yields nothing does not empty the lane;
830
+ // the candidates then flow through the DecisionModel seam (adjudicateAndWriteRows: capture gate, sensitivity
831
+ // guardrail, dedupe, identity-keyed supersession) before any row is written, and the completed run is recorded
832
+ // (#189: a zero-yield lane has no row to mark it done).
833
+ //
834
+ // A failure (extractor-parse-error or otherwise) is never cached and never a silent []: the note hash resets so
835
+ // the next pass retries, and the error is rethrown so it is counted (#188). A row of a stale version was not
836
+ // reattached above and is deliberately not put back: a bumped extraction_version must never be served as current.
837
+ for (const lane of structuredLanes(options)) {
838
+ if (laneDone(db, filePath, lane, noteHash))
839
+ continue;
840
+ try {
841
+ const candidates = await lane.extract(parsed.title, parsed.body, insertedChunks);
842
+ (0, rowStore_1.deleteStructuredRowsForSourceKind)(db, filePath, lane.kind);
843
+ const outcome = await (0, writePathAdjudication_1.adjudicateAndWriteRows)(db, {
844
+ rowKind: lane.kind,
845
+ extractionVersion: lane.version,
846
+ notePath: filePath,
847
+ sourceHash: noteHash,
848
+ // The note's front-matter date (NULL = unknown), never one resolved from an extracted phrase.
849
+ validFrom: (0, typedFacts_1.contentDateFromFrontmatter)(parsed.frontmatter),
850
+ candidates,
851
+ }, options.decisionModel);
852
+ // What this run's writes returned, not a re-read of structured_rows a concurrent writer could tear (review B1).
853
+ (0, laneRuns_1.recordLaneRun)(db, filePath, lane.kind, noteHash, lane.version, outcome.rowsWritten);
854
+ }
855
+ catch (err) {
856
+ log.error('extraction_failed', { lane: lane.label, note: path_1.default.basename(filePath) }, err);
857
+ db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
858
+ throw err;
859
+ }
860
+ }
861
+ const rows = { ids: insertedChunks.map(c => c.id), hashes: chunkHashes };
862
+ const next = appendBaseOf(filePath, raw, content, detailed, kind, cut, rows, noteHash, options);
863
+ if (next)
864
+ recordAppendBase(db, filePath, next);
865
+ }
866
+ async function indexVault(db, vaultPath, options = {}, concurrency = 5) {
867
+ // A name another OS or locale wrote can hold bytes that are not UTF-8. Read the directory as
868
+ // bytes and name those files here instead of letting the walk return a path that exists
869
+ // nowhere: for a FILE that path failed the first read and counted as an indexing error, and
870
+ // for a FOLDER `scandir` threw inside the walk, so one such name aborted the whole pass and
871
+ // not one note reached the index.
872
+ const skippedNonUtf8Names = [];
873
+ const files = await (0, walker_1.walkVault)(vaultPath, {
874
+ onNonUtf8Markdown: relPath => skippedNonUtf8Names.push(relPath),
875
+ });
876
+ const walked = files.length;
877
+ const skippedTooLarge = [];
878
+ let indexed = 0;
879
+ let errors = 0;
880
+ let fileIdx = 0;
881
+ let active = 0;
882
+ // Skipped names are reported even when they are the only Markdown in the vault: "nothing to
883
+ // index" and "nothing indexable" must not read the same.
884
+ if (walked === 0)
885
+ return { total: 0, indexed: 0, errors: 0, skippedNonUtf8Names: skippedNonUtf8Names.sort() };
886
+ console.log(`[index] starting: ${walked} files, concurrency ${concurrency}`);
887
+ return new Promise((resolve) => {
888
+ function onComplete() {
889
+ const done = indexed + errors + skippedTooLarge.length;
890
+ if (done % 50 === 0 || done === walked) {
891
+ console.log(`[index] ${done}/${walked} files (${errors} errors)`);
892
+ }
893
+ if (done === walked) {
894
+ resolve({
895
+ total: walked - skippedTooLarge.length,
896
+ indexed,
897
+ errors,
898
+ skippedNonUtf8Names: skippedNonUtf8Names.sort(),
899
+ ...(skippedTooLarge.length > 0 ? { skippedTooLarge: skippedTooLarge.sort() } : {}),
900
+ });
901
+ }
902
+ else {
903
+ next();
904
+ }
905
+ }
906
+ function next() {
907
+ while (active < concurrency && fileIdx < walked) {
908
+ const filePath = files[fileIdx++];
909
+ active++;
910
+ indexFile(db, filePath, options).then(() => {
911
+ indexed++;
912
+ active--;
913
+ onComplete();
914
+ }, (err) => {
915
+ active--;
916
+ if (err instanceof NoteTooLargeError) {
917
+ skippedTooLarge.push(path_1.default.relative(vaultPath, filePath));
918
+ log.warn('note_too_large', { file: path_1.default.relative(vaultPath, filePath) }, err);
919
+ }
920
+ else {
921
+ errors++;
922
+ log.error('index_file_failed', { file: path_1.default.relative(vaultPath, filePath) }, err);
923
+ }
924
+ onComplete();
925
+ });
926
+ }
927
+ }
928
+ next();
929
+ });
930
+ }
931
+ /**
932
+ * Remove the facts an earlier build made of the engine's own daily-note headers (dailyNote.ts), and say how many. An
933
+ * index written then holds one for every daily note, and a note whose bytes did not change is never rebuilt, so the row
934
+ * would stay and be served (retrieve_memory answers it as a current fact). The server calls this when it starts, the
935
+ * indexer when it passes over an unchanged daily note (it hands over the text it has just read); `filePath` narrows it
936
+ * to one note.
937
+ *
938
+ * Nothing a person wrote is touched. A row goes only when it looks like what that indexer made of the header (a fact
939
+ * named `updated` that no ledger event made, whose text is the note's own day as a Date) AND the note is one the engine
940
+ * wrote: the front matter at the start of the file, read here, carries the engine's marker or is the complete front
941
+ * matter the engine wrote before it had one (isEngineDailyNote). A person's note in the same folder with a natural
942
+ * `updated:`, a note whose file is gone, or one that cannot be read, is not told to be the engine's, and its row stays.
943
+ */
944
+ function removeDailyHeaderFacts(db, filePath, content) {
945
+ if (filePath !== undefined && !(0, dailyNote_1.isDailyNotePath)(filePath))
946
+ return 0;
947
+ const rows = (filePath === undefined
948
+ ? db.prepare('SELECT id, source_path, object_text FROM facts WHERE predicate = ? AND event_id IS NULL').all(dailyNote_1.DAILY_NOTE_HEADER_KEY)
949
+ : db.prepare('SELECT id, source_path, object_text FROM facts WHERE predicate = ? AND event_id IS NULL AND source_path = ?').all(dailyNote_1.DAILY_NOTE_HEADER_KEY, filePath));
950
+ const byNote = new Map();
951
+ for (const row of rows) {
952
+ if (typeof row.source_path !== 'string' || !(0, dailyNote_1.isEarlierHeaderText)(row.source_path, row.object_text))
953
+ continue;
954
+ byNote.set(row.source_path, [...(byNote.get(row.source_path) ?? []), row.id]);
955
+ }
956
+ const ids = [];
957
+ for (const [note, noteIds] of byNote) {
958
+ const head = filePath !== undefined && content !== undefined ? content : readHead(note);
959
+ if (head !== null && (0, dailyNote_1.isEngineDailyNote)(note, head))
960
+ ids.push(...noteIds);
961
+ }
962
+ if (ids.length === 0)
963
+ return 0;
964
+ const drop = db.prepare('DELETE FROM facts WHERE id = ?');
965
+ db.transaction(() => {
966
+ for (const id of ids)
967
+ drop.run(id);
968
+ })();
969
+ return ids.length;
970
+ }
971
+ const HEAD_BYTES = 4096;
972
+ /** The start of a file; null when it cannot be read (a note that is gone proves nothing). */
973
+ function readHead(file) {
974
+ let fd;
975
+ try {
976
+ fd = (0, fs_2.openSync)(file, 'r');
977
+ const buffer = Buffer.alloc(HEAD_BYTES);
978
+ const read = (0, fs_2.readSync)(fd, buffer, 0, HEAD_BYTES, 0);
979
+ return buffer.subarray(0, read).toString('utf8');
980
+ }
981
+ catch {
982
+ return null;
983
+ }
984
+ finally {
985
+ if (fd !== undefined)
986
+ (0, fs_2.closeSync)(fd);
987
+ }
988
+ }