renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,672 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.ImportStorageLimitError = void 0;
7
- exports.applyImportPlan = applyImportPlan;
8
- exports.isEmptyLeftover = isEmptyLeftover;
9
- const promises_1 = __importDefault(require("fs/promises"));
10
- const fs_1 = __importDefault(require("fs"));
11
- const path_1 = __importDefault(require("path"));
12
- const client_1 = require("../db/client");
13
- const schema_1 = require("../db/schema");
14
- const hasher_1 = require("../indexer/hasher");
15
- const vectorIndex_1 = require("../embeddings/vectorIndex");
16
- const confinedWrite_1 = require("../memory/confinedWrite");
17
- const durableAppend_1 = require("../memory/durableAppend");
18
- const lifecycle_1 = require("../memory/lifecycle");
19
- const secretGuard_1 = require("../extraction/secretGuard");
20
- const secretScreen_1 = require("./secretScreen");
21
- /**
22
- * The write layer.
23
- *
24
- * Two layers are written deliberately:
25
- *
26
- * 1. **Canonical** — Markdown notes under an additive namespace plus ledger events.
27
- * This is what the engine treats as durable: the index is derived and rebuildable,
28
- * the ledger survives `rm index.db`, and the workspace's own reads (fact list and
29
- * graph) are built only from lifecycle facts.
30
- * 2. **Derived index rows** — the source's own parse of notes/chunks/entities, copied
31
- * so the workspace shows real counts and lexical recall works **without any
32
- * re-chunking, re-embedding or re-extraction**. This is a deliberate exception to
33
- * "the index is derived": the derived rows are written only *after* the canonical
34
- * bytes exist, and each copied note's `note_hash` is the hash of the exact bytes
35
- * written, which is the engine's own condition for "unchanged — skip". Vectors are
36
- * never transferred (an embedding is model-bound) and no embedding call is made here:
37
- * an index whose vectors are in a model's space is marked unfinished in the same commit
38
- * (`markVectorsUnfinished`), so the workspace's own model embeds exactly the copied
39
- * chunks, once, before semantic search reads the index again.
40
- */
41
- /** A final measured-size refusal. applyImportPlan compensates canonical and derived writes before rethrowing it. */
42
- class ImportStorageLimitError extends Error {
43
- }
44
- exports.ImportStorageLimitError = ImportStorageLimitError;
45
- const PHASE_DESCRIPTION = {
46
- notes: 'writing the imported notes failed',
47
- 'ledger-read': 'reading the destination ledger failed',
48
- ledger: 'appending the carried events to the ledger failed',
49
- index: 'the destination index could not be opened or migrated',
50
- replay: 'the ledger replay rejected the carried events',
51
- copy: 'copying the source parse into the destination index failed',
52
- };
53
- async function applyImportPlan(plan, _bundle, options) {
54
- // RENOR_SECRET_GUARD (default ON): a plan that still holds a note or fact that looks like a secret is refused
55
- // whole, here, before the first directory, file, ledger event or index row. buildImportPlan leaves such items
56
- // out, so this catches a plan built with the guard off, or by another caller.
57
- if ((0, secretGuard_1.secretGuardEnabled)())
58
- (0, secretScreen_1.refuseFlaggedPlan)(plan);
59
- const memoryDir = path_1.default.resolve(options.memoryDir);
60
- const dbPath = options.dbPath ?? path_1.default.join(memoryDir, '.memory-index', 'index.db');
61
- const ledgerRelPath = options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL;
62
- const ledgerPath = path_1.default.join(memoryDir, ...ledgerRelPath.split('/'));
63
- let notesWritten = 0;
64
- let notesUnchanged = 0;
65
- let notesConflicting = 0;
66
- /** Destination paths that already hold different content: never written, never indexed. */
67
- const skippedConflicts = new Set();
68
- const conflicts = [];
69
- // Compensation state. Every path in `created` was absent before this invocation, so
70
- // removing it cannot lose memory the operator or another writer owns.
71
- const created = { files: [], dirs: [] };
72
- let phase = 'notes';
73
- let priorLedger = null;
74
- let appendedLedger = null;
75
- // Set immediately before this invocation's first append. A failed read or parse must
76
- // never enter ledger compensation: no ledger bytes have been touched yet.
77
- let ledgerTouched = false;
78
- let db = null;
79
- let committed = false;
80
- let ledgerReplayed = 0;
81
- let eventsAppended = 0;
82
- let eventsAlreadyPresent = 0;
83
- let indexRows = {
84
- notesInserted: 0,
85
- notesUpdated: 0,
86
- chunksInserted: 0,
87
- chunksAlreadyPresent: 0,
88
- entitiesInserted: 0,
89
- entitiesAlreadyPresent: 0,
90
- };
91
- try {
92
- // A first vault import can target a workspace that has no memory directory
93
- // yet. Create the root before creating tracked additive subdirectories.
94
- ensureDirTracked(memoryDir, memoryDir, created);
95
- for (const planned of plan.notes) {
96
- const target = path_1.default.join(memoryDir, ...planned.relPath.split('/'));
97
- const dir = path_1.default.dirname(target);
98
- if (!isInside(memoryDir, target)) {
99
- throw new Error(`refusing to write outside the memory directory: ${planned.relPath}`);
100
- }
101
- assertNoLinkBelow(memoryDir, dir);
102
- ensureDirTracked(memoryDir, dir, created);
103
- // Exclusive create: this run owns the file only if this run is what brought it into
104
- // existence. A plain `writeFile` decides ownership from an `ENOENT` read that the
105
- // write does not repeat, so anything appearing in that window is overwritten and then
106
- // deleted by the compensation — and `writeFile` follows a symlink, so a dangling link
107
- // at the planned path would have the import write outside the memory directory and
108
- // then remove the link. `wx` makes the create atomic, so ownership is observed rather
109
- // than assumed, and only a path this run created is ever recorded for removal. An empty
110
- // leftover of an earlier import killed between that create and its write is replaced
111
- // instead of reported as a conflict (openEmptyLeftover); the containment check below
112
- // still runs on whichever handle it gets, before anything is written or recorded.
113
- const handle = (await openExclusive(target)) ?? (await openEmptyLeftover(target));
114
- if (handle) {
115
- // A directory swapped for a link after assertNoLinkBelow would have the open land elsewhere: ask the kernel where
116
- // the file is before anything is written or recorded (src/memory/confinedWrite.ts).
117
- try {
118
- await (0, confinedWrite_1.assertHandleInside)(handle, await promises_1.default.realpath(memoryDir), planned.relPath);
119
- }
120
- catch (err) {
121
- await handle.close().catch(() => undefined);
122
- throw err;
123
- }
124
- // From here the file exists because of THIS run, so it is this run's to remove if
125
- // anything later fails.
126
- created.files.push(target);
127
- try {
128
- options.onNoteCreated?.(planned);
129
- await handle.writeFile(planned.content, 'utf8');
130
- // Durable before the import is acknowledged, like every other acknowledged Markdown write (adv-crashsafe-1012 F3).
131
- await handle.sync();
132
- }
133
- finally {
134
- await handle.close();
135
- }
136
- notesWritten += 1;
137
- continue;
138
- }
139
- // Something is already at the path, and the import is additive: it is never written
140
- // to. A path this run did not create — including a symlink, which must not be written
141
- // through even when it points back inside the memory directory — is a conflict the
142
- // operator resolves, and it is skipped for the index too.
143
- const existing = await promises_1.default.lstat(target);
144
- if (!existing.isFile()) {
145
- notesConflicting += 1;
146
- skippedConflicts.add(target);
147
- conflicts.push(planned.relPath);
148
- continue;
149
- }
150
- const onDisk = await promises_1.default.readFile(target, 'utf8');
151
- if (onDisk === planned.content) {
152
- options.onNoteUnchanged?.(planned);
153
- notesUnchanged += 1;
154
- continue;
155
- }
156
- notesConflicting += 1;
157
- skippedConflicts.add(target);
158
- conflicts.push(planned.relPath);
159
- }
160
- phase = 'ledger-read';
161
- // Snapshot the durable authority before any append. Only ENOENT means no ledger;
162
- // permission and other read failures propagate so an existing ledger is never
163
- // mistaken for absence and later removed by rollback. Parse IDs from the same bytes
164
- // that compensation will restore, avoiding a second read with a different outcome.
165
- // A ledger reached through a link, or a link itself, is refused before a byte is read from or written to it.
166
- await (0, confinedWrite_1.assertConfinedTarget)(memoryDir, path_1.default.relative(memoryDir, ledgerPath));
167
- priorLedger = readFileOrNull(ledgerPath);
168
- const existingLedgerIds = new Set((0, lifecycle_1.parseLedgerEvents)(priorLedger ?? '').map(event => event.id.toUpperCase()));
169
- // `appendLedgerEvent` creates the ledger's directory chain itself. Record the
170
- // directories that are missing now, so a rollback cannot leave an empty tree behind.
171
- for (const dir of missingDirsUnder(memoryDir, path_1.default.dirname(ledgerPath)))
172
- created.dirs.push(dir);
173
- try {
174
- for (const event of plan.events) {
175
- const eventKey = event.id.toUpperCase();
176
- if (existingLedgerIds.has(eventKey)) {
177
- eventsAlreadyPresent += 1;
178
- continue;
179
- }
180
- // From this point the durable bytes may change, so and only so can rollback restore
181
- // the prior snapshot (or remove a ledger this invocation created from an absent path).
182
- phase = 'ledger';
183
- ledgerTouched = true;
184
- await (0, lifecycle_1.appendLedgerEvent)(memoryDir, ledgerRelPath, toLedgerEvent(event));
185
- existingLedgerIds.add(eventKey);
186
- eventsAppended += 1;
187
- }
188
- }
189
- finally {
190
- // Capture what this run actually left behind even when an append rejects. The
191
- // compare-before-restore guard must compare against these bytes, not `null`, after
192
- // an earlier event in the same plan was appended successfully.
193
- if (ledgerTouched)
194
- appendedLedger = readFileOrNull(ledgerPath);
195
- }
196
- phase = 'index';
197
- db = (0, client_1.openDb)(dbPath);
198
- (0, schema_1.runMigrations)(db);
199
- // The append and the projection are ONE unit, in both directions.
200
- //
201
- // `replayLedger` applies events in a bare loop and each `applyEvent` commits its own
202
- // transaction, so a mid-loop rejection would leave the already-applied facts
203
- // committed. Restoring the ledger on top of that would remove the events while their
204
- // facts remain — and the engine's global preflight (K13) fails closed when the index
205
- // holds lifecycle facts whose event id is absent from the ledger. The result would be
206
- // a workspace that refuses to serve, which is worse than the rejection itself.
207
- // Wrapping the replay in one outer transaction makes the rollback real: the engine's
208
- // own `db.transaction` calls become savepoints inside it.
209
- phase = 'replay';
210
- db.exec('BEGIN IMMEDIATE');
211
- try {
212
- // The lifecycle projection is applied through the engine's own replay, so the
213
- // imported facts and the destination's existing facts obey one write path.
214
- ledgerReplayed = await (0, lifecycle_1.replayLedger)(db, memoryDir, ledgerRelPath);
215
- phase = 'copy';
216
- if (!options.canonicalOnly) {
217
- indexRows = copyParseRows(db, memoryDir, plan, skippedConflicts);
218
- // The copied chunks have no vector. The mark lands in this commit, so it and the chunks land together or not at
219
- // all; the next engine with the index's model embeds every chunk that has none, once (a running service starts
220
- // that job when the import returns).
221
- if (indexRows.chunksInserted > 0)
222
- (0, vectorIndex_1.markVectorsUnfinished)(db);
223
- }
224
- // Canonical Markdown is now complete, but neither its index projection nor the files have committed as a unit.
225
- // This is the last point at which a measured workspace ceiling can reject and compensate every added byte.
226
- await options.validateBeforeCommit?.();
227
- db.exec('COMMIT');
228
- committed = true;
229
- }
230
- catch (err) {
231
- // A failed ROLLBACK must not hide the failure that caused it.
232
- try {
233
- db.exec('ROLLBACK');
234
- }
235
- catch { /* the transaction is already unwound */ }
236
- throw err;
237
- }
238
- return {
239
- namespace: plan.namespace,
240
- notesWritten,
241
- notesUnchanged,
242
- notesConflicting,
243
- conflicts,
244
- eventsAppended,
245
- eventsAlreadyPresent,
246
- ledgerReplayed,
247
- indexRows,
248
- destination: measureDestination(db),
249
- };
250
- }
251
- catch (err) {
252
- // Past COMMIT the projection is durable and there is nothing to roll back; saying that
253
- // is the honest report, and it must not read like an ordinary failed import.
254
- if (committed)
255
- throw committedFailure(err);
256
- // One compensation for the whole unit. The projection is already unwound (rolled back
257
- // or never started), so what remains is the canonical layer this run added: its own
258
- // note bytes, its own directories, and the ledger bytes it appended to. Files the run
259
- // did not create — unchanged notes, conflicts, the operator's own files — are never
260
- // touched, and pruning is limited to directories that are empty afterwards.
261
- const restore = ledgerTouched
262
- ? await restoreLedger(ledgerPath, priorLedger, appendedLedger)
263
- : { restored: false };
264
- // A ledger that could not be restored still holds its carried events, and those events
265
- // are canonical: removing the notes they describe would leave a ledger pointing at
266
- // memory that no longer exists. Keep the notes so the canonical layer stays
267
- // self-consistent and recoverable, and report the residual instead of destroying it.
268
- const notesRemoved = ledgerTouched && !restore.restored ? 0 : compensateCreatedPaths(created);
269
- if (err instanceof ImportStorageLimitError && (!ledgerTouched || restore.restored))
270
- throw err;
271
- throw new Error(rollbackMessage({ phase, err, notesRemoved, ledgerTouched, restore, eventsAppended }));
272
- }
273
- finally {
274
- try {
275
- db?.close();
276
- }
277
- catch (closeErr) {
278
- // A close failure after COMMIT is a post-commit failure: the writes are durable and
279
- // no rollback happened. Before COMMIT the primary failure is already propagating and
280
- // would be replaced by throwing here, so this one is dropped rather than allowed to
281
- // hide the real cause.
282
- if (committed)
283
- throw committedFailure(closeErr);
284
- }
285
- }
286
- }
287
- /**
288
- * `open(path, 'wx')`, or `null` when something already exists at the path.
289
- * Exposed as a function so the exclusivity is one decision in one place, not a flag each
290
- * caller has to remember.
291
- */
292
- async function openExclusive(target) {
293
- try {
294
- return await promises_1.default.open(target, 'wx', 0o600);
295
- }
296
- catch (err) {
297
- if (err.code === 'EEXIST')
298
- return null;
299
- throw err;
300
- }
301
- }
302
- /**
303
- * Opens an empty leftover of an earlier import for writing, or returns `null` when the path holds anything else. An
304
- * earlier import created each note before writing it, so a kill between the two left an empty file that no journal
305
- * named; a retry replaces it, holding nothing to lose, instead of reporting a conflict. The open never creates and never
306
- * follows a link (O_NOFOLLOW), a hard-linked file is refused (a write would reach its other name too), and the size is
307
- * checked on the open descriptor, so the file written is the file that was checked. The caller still runs the
308
- * containment check on the handle before anything is written or recorded. The check and the open are two calls, so a file
309
- * another writer put at that exact instant would be written; an import is additive and runs on the owner's own workspace,
310
- * so that window is accepted.
311
- */
312
- async function openEmptyLeftover(target) {
313
- if (!isEmptyLeftover(await promises_1.default.lstat(target)))
314
- return null;
315
- let handle;
316
- try {
317
- handle = await promises_1.default.open(target, fs_1.default.constants.O_WRONLY | fs_1.default.constants.O_NOFOLLOW | fs_1.default.constants.O_NONBLOCK);
318
- }
319
- catch (err) {
320
- if (['ELOOP', 'EACCES'].includes(err.code ?? ''))
321
- return null;
322
- throw err;
323
- }
324
- const opened = await handle.stat();
325
- if (!isEmptyLeftover(opened) || opened.nlink !== 1) {
326
- await handle.close();
327
- return null;
328
- }
329
- return handle;
330
- }
331
- /** An empty regular file (not a symlink, not a directory): what an interrupted import leaves at a note's path. */
332
- function isEmptyLeftover(stat) {
333
- return stat.isFile() && stat.size === 0;
334
- }
335
- /** A failure after the import committed: durable writes that were deliberately not rolled back. */
336
- function committedFailure(err) {
337
- const detail = err instanceof Error ? err.message : String(err);
338
- return new Error('import committed but the run failed afterwards, so no rollback was performed and the imported '
339
- + `notes, ledger events and index rows remain as written (${detail})`);
340
- }
341
- /**
342
- * The failure text. It names the step that actually failed and what was undone, so an
343
- * operator is never told "the ledger replay rejected the carried events" for a chunk
344
- * insert or a bad destination index, and never told the ledger was restored when it was not.
345
- */
346
- function rollbackMessage(args) {
347
- const { phase, err, notesRemoved, ledgerTouched, restore, eventsAppended } = args;
348
- const detail = err instanceof Error ? err.message : String(err);
349
- const compensated = [];
350
- if (notesRemoved > 0)
351
- compensated.push(`the ${notesRemoved} note file(s) this run created were removed`);
352
- if (ledgerTouched)
353
- compensated.push(ledgerOutcome(restore, eventsAppended));
354
- compensated.push('no facts were applied');
355
- return `import rolled back: ${PHASE_DESCRIPTION[phase]}, so ${compensated.join(', ')} (${detail})`;
356
- }
357
- /**
358
- * What happened to the ledger, stated exactly.
359
- *
360
- * The two failure branches are different states and must not be described with one
361
- * sentence: with events actually appended, those events are canonical and will replay, so
362
- * the notes are kept to keep the two layers consistent; with nothing appended, the ledger's
363
- * state is simply unverified (an append that failed part-way may still have written bytes),
364
- * and the notes are kept rather than removed against a ledger nobody has read back.
365
- */
366
- function ledgerOutcome(restore, eventsAppended) {
367
- if (restore.restored)
368
- return 'the ledger was restored to its previous contents';
369
- const cause = restore.error ?? 'unknown error';
370
- if (eventsAppended > 0) {
371
- return 'the LEDGER COULD NOT BE RESTORED'
372
- + ` (${cause}), so the ${eventsAppended} event(s) this run appended remain and WILL replay: the note`
373
- + ' files this run created were left in place so the canonical notes and ledger stay consistent,'
374
- + ' and the destination needs its own repair';
375
- }
376
- return 'the LEDGER COULD NOT BE RESTORED'
377
- + ` (${cause}) and its state could not be verified, so the note files this run created were left in`
378
- + ' place rather than removed against a ledger that was not read back; the destination needs its own repair';
379
- }
380
- /**
381
- * Refuse a destination directory whose chain under the memory directory holds a symbolic link: the writes below it would
382
- * land wherever the link points. The memory directory itself may be a link, it is where the operator put the vault. A
383
- * directory that does not exist yet is fine.
384
- */
385
- function assertNoLinkBelow(root, dir) {
386
- const relative = path_1.default.relative(path_1.default.resolve(root), path_1.default.resolve(dir));
387
- if (relative === '' || relative.startsWith('..') || path_1.default.isAbsolute(relative))
388
- return;
389
- let current = path_1.default.resolve(root);
390
- for (const segment of relative.split(path_1.default.sep)) {
391
- current = path_1.default.join(current, segment);
392
- let st;
393
- try {
394
- st = fs_1.default.lstatSync(current);
395
- }
396
- catch (err) {
397
- if (err.code === 'ENOENT')
398
- return;
399
- throw err;
400
- }
401
- if (st.isSymbolicLink()) {
402
- throw new Error(`refusing to write below ${JSON.stringify(path_1.default.relative(path_1.default.resolve(root), current))}: it is a symbolic link`);
403
- }
404
- }
405
- }
406
- /**
407
- * `mkdir -p` that records every directory THIS run had to create.
408
- *
409
- * `fs.mkdir(dir, {recursive:true})` reports only the first directory it created, which is
410
- * not enough to prune the chain again; and compensation must be able to tell a directory
411
- * it made from one that was already there.
412
- */
413
- function ensureDirTracked(root, dir, created) {
414
- const base = path_1.default.resolve(root);
415
- const target = path_1.default.resolve(dir);
416
- const relative = path_1.default.relative(base, target);
417
- if (relative === '') {
418
- // The memory directory itself: create it when absent, but never claim it for removal.
419
- fs_1.default.mkdirSync(target, { recursive: true, mode: 0o700 });
420
- return;
421
- }
422
- if (relative.startsWith('..') || path_1.default.isAbsolute(relative)) {
423
- fs_1.default.mkdirSync(target, { recursive: true });
424
- return;
425
- }
426
- for (const missing of missingDirsUnder(base, target)) {
427
- try {
428
- fs_1.default.mkdirSync(missing, { mode: 0o700 });
429
- created.dirs.push(missing);
430
- }
431
- catch (err) {
432
- // Created between the check and the mkdir: it is not this run's to remove.
433
- if (err.code === 'EEXIST')
434
- continue;
435
- throw err;
436
- }
437
- }
438
- }
439
- /** Directory chain under `root` that does not exist yet, outermost first. */
440
- function missingDirsUnder(root, dir) {
441
- const base = path_1.default.resolve(root);
442
- const target = path_1.default.resolve(dir);
443
- const relative = path_1.default.relative(base, target);
444
- if (relative === '' || relative.startsWith('..') || path_1.default.isAbsolute(relative))
445
- return [];
446
- const missing = [];
447
- let current = base;
448
- for (const segment of relative.split(path_1.default.sep)) {
449
- current = path_1.default.join(current, segment);
450
- if (!fs_1.default.existsSync(current))
451
- missing.push(current);
452
- }
453
- return missing;
454
- }
455
- /**
456
- * Remove what this run created: its own note files, then its own directories deepest-first
457
- * and only while they are empty. Nothing recursive is ever removed — a directory that has
458
- * picked up any other entry is left exactly as it is.
459
- */
460
- function compensateCreatedPaths(created) {
461
- let removed = 0;
462
- for (const file of created.files) {
463
- try {
464
- fs_1.default.unlinkSync(file);
465
- removed += 1;
466
- }
467
- catch {
468
- // Already gone: the compensation is idempotent.
469
- }
470
- }
471
- const dirs = [...created.dirs].sort((a, b) => b.split(path_1.default.sep).length - a.split(path_1.default.sep).length || b.length - a.length);
472
- for (const dir of dirs) {
473
- try {
474
- if (fs_1.default.readdirSync(dir).length === 0)
475
- fs_1.default.rmdirSync(dir);
476
- }
477
- catch {
478
- // Not empty, or already gone: leave it.
479
- }
480
- }
481
- return removed;
482
- }
483
- function readFileOrNull(filePath) {
484
- try {
485
- return fs_1.default.readFileSync(filePath, 'utf8');
486
- }
487
- catch (err) {
488
- if (err.code === 'ENOENT')
489
- return null;
490
- throw err;
491
- }
492
- }
493
- /**
494
- * Exact restore of the prior ledger bytes, or removal when it did not exist before.
495
- * Returns the OUTCOME rather than swallowing it: a run that could not restore the ledger
496
- * still holds the carried events, and must say so instead of claiming a clean rollback.
497
- */
498
- function restoreLedger(ledgerPath, prior, appended) {
499
- // Under the lock every ledger writer takes (appends, and the erasure's rewrite in lifecycle.ts), so no writer can
500
- // change the ledger between the checks below and the restore.
501
- return (0, durableAppend_1.withAppendLock)(ledgerPath, async () => restoreLedgerLocked(ledgerPath, prior, appended));
502
- }
503
- function restoreLedgerLocked(ledgerPath, prior, appended) {
504
- try {
505
- // Compensation owns only the bytes it appended. If another lifecycle writer changed the
506
- // ledger afterwards (especially by replacing a fact with an erase tombstone), restoring
507
- // this import's older snapshot would resurrect that writer's deleted canonical content.
508
- if (readFileOrNull(ledgerPath) !== appended) {
509
- return { restored: false, error: 'the ledger changed while this import ran (another save, or an erasure); refusing to overwrite newer canonical memory' };
510
- }
511
- // The snapshot was read before the appends, without the lock. Only appends may have happened since: then the snapshot
512
- // is a prefix of what this run left. Anything else (an erasure rewrote the ledger in between) makes the snapshot stale,
513
- // and restoring it would write erased content back.
514
- if (prior !== null && (appended === null || !appended.startsWith(prior))) {
515
- return { restored: false, error: 'the ledger changed while this import ran (another save, or an erasure); refusing to overwrite newer canonical memory' };
516
- }
517
- if (prior === null) {
518
- if (fs_1.default.existsSync(ledgerPath))
519
- fs_1.default.unlinkSync(ledgerPath);
520
- return { restored: !fs_1.default.existsSync(ledgerPath) };
521
- }
522
- fs_1.default.writeFileSync(ledgerPath, prior, 'utf8');
523
- return { restored: fs_1.default.readFileSync(ledgerPath, 'utf8') === prior };
524
- }
525
- catch (err) {
526
- return { restored: false, error: err instanceof Error ? err.message : String(err) };
527
- }
528
- }
529
- function toLedgerEvent(event) {
530
- return {
531
- id: event.id,
532
- at: event.at,
533
- kind: event.kind,
534
- subject: event.subject,
535
- predicate: event.predicate,
536
- object: event.object,
537
- closedEvent: null,
538
- source: event.source,
539
- scope: event.scope,
540
- };
541
- }
542
- /**
543
- * Copy the source's parse of notes, chunks and entities into the destination index.
544
- * Facts and relations are deliberately NOT copied: the dashboard's fact list and graph
545
- * read lifecycle facts only, and relations carry no foreign key (they would fail
546
- * silently). Note-derived entities whose `source_note` is the lifecycle sentinel are
547
- * skipped so the destination's own lifecycle entity is never duplicated.
548
- */
549
- function copyParseRows(db, memoryDir, plan, skippedConflicts) {
550
- const counters = {
551
- notesInserted: 0,
552
- notesUpdated: 0,
553
- chunksInserted: 0,
554
- chunksAlreadyPresent: 0,
555
- entitiesInserted: 0,
556
- entitiesAlreadyPresent: 0,
557
- };
558
- const findNote = db.prepare('SELECT note_hash FROM notes WHERE path = ?');
559
- const insertNote = db.prepare(`INSERT INTO notes (path, title, kind, note_hash, modified_at, frontmatter_json)
560
- VALUES (@path, @title, @kind, @noteHash, @modifiedAt, @frontmatterJson)`);
561
- const updateNote = db.prepare(`UPDATE notes SET title = @title, kind = @kind, note_hash = @noteHash,
562
- modified_at = @modifiedAt, frontmatter_json = @frontmatterJson WHERE path = @path`);
563
- const countChunks = db.prepare('SELECT count(*) AS c FROM chunks WHERE note_path = ?');
564
- const insertChunk = db.prepare(`INSERT INTO chunks (note_path, heading_path, text, start_line, end_line, token_count, chunk_hash)
565
- VALUES (@notePath, @headingPath, @text, @startLine, @endLine, @tokenCount, @chunkHash)`);
566
- const findEntity = db.prepare('SELECT id FROM entities WHERE canonical_name = ? AND type = ? AND source_note = ?');
567
- const insertEntity = db.prepare(`INSERT INTO entities (type, canonical_name, aliases_json, source_note, confidence, updated_at)
568
- VALUES (@type, @canonicalName, @aliasesJson, @sourceNote, @confidence, @updatedAt)`);
569
- const run = db.transaction(() => {
570
- for (const planned of plan.notes) {
571
- const target = path_1.default.join(memoryDir, ...planned.relPath.split('/'));
572
- // A path that was not written (conflict, or a file the operator owns) gets no
573
- // index rows: otherwise the row would carry the source's hash and chunk text for
574
- // bytes that are not on disk, which both breaks search honesty and makes the next
575
- // index pass see a change and re-chunk and re-embed the note.
576
- if (skippedConflicts.has(target))
577
- continue;
578
- // The note row must hash the bytes on disk: this is exactly the engine's
579
- // "unchanged, skip" condition, so the note is never re-chunked or re-embedded.
580
- const noteHash = (0, hasher_1.hashContent)(planned.content);
581
- const modifiedAt = safeMtime(target);
582
- const note = planned.note;
583
- const row = {
584
- path: target,
585
- title: note.title || path_1.default.basename(planned.relPath),
586
- kind: note.kind,
587
- noteHash,
588
- modifiedAt,
589
- frontmatterJson: JSON.stringify(note.frontmatter ?? {}),
590
- };
591
- const existing = findNote.get(target);
592
- if (!existing) {
593
- insertNote.run(row);
594
- counters.notesInserted += 1;
595
- }
596
- else if (existing.note_hash !== noteHash) {
597
- // The file on disk is authoritative for the row, so re-sync rather than leave a
598
- // stale hash that would make a later index pass re-process this note.
599
- updateNote.run(row);
600
- counters.notesUpdated += 1;
601
- }
602
- const chunkCount = countChunks.get(target).c;
603
- if (chunkCount > 0) {
604
- counters.chunksAlreadyPresent += note.chunks.length;
605
- }
606
- else {
607
- for (let i = 0; i < note.chunks.length; i += 1) {
608
- const chunk = note.chunks[i];
609
- const lines = planned.chunkLines[i] ?? { startLine: 0, endLine: 0 };
610
- insertChunk.run({
611
- notePath: target,
612
- headingPath: chunk.headingPath,
613
- text: chunk.text,
614
- startLine: lines.startLine,
615
- endLine: lines.endLine,
616
- tokenCount: chunk.tokenCount,
617
- chunkHash: (0, hasher_1.hashContent)(chunk.text),
618
- });
619
- counters.chunksInserted += 1;
620
- }
621
- }
622
- for (const entity of note.entities) {
623
- if (!entity.canonicalName.trim())
624
- continue;
625
- const found = findEntity.get(entity.canonicalName, entity.type, target);
626
- if (found) {
627
- counters.entitiesAlreadyPresent += 1;
628
- continue;
629
- }
630
- insertEntity.run({
631
- type: entity.type || 'note',
632
- canonicalName: entity.canonicalName,
633
- aliasesJson: JSON.stringify(entity.aliases ?? []),
634
- sourceNote: target,
635
- confidence: typeof entity.confidence === 'number' ? entity.confidence : 1,
636
- updatedAt: new Date().toISOString(),
637
- });
638
- counters.entitiesInserted += 1;
639
- }
640
- }
641
- });
642
- run();
643
- return counters;
644
- }
645
- function safeMtime(filePath) {
646
- try {
647
- return fs_1.default.statSync(filePath).mtime.toISOString();
648
- }
649
- catch {
650
- return new Date().toISOString();
651
- }
652
- }
653
- function isInside(root, candidate) {
654
- const rel = path_1.default.relative(root, candidate);
655
- return rel !== '' && !rel.startsWith('..') && !path_1.default.isAbsolute(rel);
656
- }
657
- function measureDestination(db) {
658
- const scalar = (sql) => db.prepare(sql).get().c;
659
- try {
660
- return {
661
- notes: scalar('SELECT count(*) AS c FROM notes'),
662
- chunks: scalar('SELECT count(*) AS c FROM chunks'),
663
- entities: scalar('SELECT count(*) AS c FROM entities'),
664
- facts: scalar('SELECT count(*) AS c FROM facts'),
665
- currentFacts: scalar(`SELECT count(*) AS c FROM facts WHERE valid_to IS NULL AND source_path = '_mcp_'`),
666
- ftsRows: scalar('SELECT count(*) AS c FROM chunks_fts'),
667
- };
668
- }
669
- catch {
670
- return null;
671
- }
672
- }