renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,1078 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.BACKGROUND_INDEX_FIRST_DELAY_MS = exports.UNFINISHED_RETRY_MS = exports.UNFINISHED_FILES = exports.TIMESTAMP_TICK_MS = exports.FULL_VAULT_CHECK_MS = exports.INDEX_FILE_CONCURRENCY = exports.ENTITY_LOOKUP_SURFACE = exports.QUERY_CACHE_MAX_BYTES = exports.QUERY_CACHE_MAX_ENTRIES = exports.DEFAULT_EMBED_BUDGETS = void 0;
7
- exports.resolveEmbedBudgets = resolveEmbedBudgets;
8
- exports.readEmbeddingTableDims = readEmbeddingTableDims;
9
- exports.queryEmbeddingCachePath = queryEmbeddingCachePath;
10
- exports.createEmbeddingProvider = createEmbeddingProvider;
11
- exports.attachEngine = attachEngine;
12
- exports.semanticReady = semanticReady;
13
- exports.keywordIndexReady = keywordIndexReady;
14
- exports.engineRebuildEmbeddings = engineRebuildEmbeddings;
15
- exports.runningEmbeddingRebuild = runningEmbeddingRebuild;
16
- exports.clientPath = clientPath;
17
- exports.resolveEngineEvidence = resolveEngineEvidence;
18
- exports.resolveEngineDigests = resolveEngineDigests;
19
- exports.engineSearchMemory = engineSearchMemory;
20
- exports.engineSearchText = engineSearchText;
21
- exports.engineLookupEntity = engineLookupEntity;
22
- exports.engineStatus = engineStatus;
23
- exports.engineIndex = engineIndex;
24
- exports.engineIndexChanged = engineIndexChanged;
25
- exports.engineFinishIndexInBackground = engineFinishIndexInBackground;
26
- exports.runningBackgroundIndex = runningBackgroundIndex;
27
- exports.hasIndexWorkToFinish = hasIndexWorkToFinish;
28
- exports.engineRememberNote = engineRememberNote;
29
- const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
30
- const invisibleText_1 = require("../memory/invisibleText");
31
- const fs_1 = __importDefault(require("fs"));
32
- const crypto_1 = __importDefault(require("crypto"));
33
- const path_1 = __importDefault(require("path"));
34
- const readCache_1 = require("../db/readCache");
35
- const statements_1 = require("../db/statements");
36
- const vectorIndex_1 = require("../embeddings/vectorIndex");
37
- const space_1 = require("../embeddings/space");
38
- const rebuild_1 = require("../embeddings/rebuild");
39
- const openai_1 = require("../embeddings/openai");
40
- const local_1 = require("../embeddings/local");
41
- const vertex_1 = require("../embeddings/vertex");
42
- const poolFetch_1 = require("../net/poolFetch");
43
- const callPhases_1 = require("../callPhases");
44
- const caching_1 = require("../embeddings/caching");
45
- const interactive_1 = require("../embeddings/interactive");
46
- const coalescing_1 = require("../embeddings/coalescing");
47
- const readerLayout_1 = require("../retrieval/readerLayout");
48
- const orchestrator_1 = require("../retrieval/orchestrator");
49
- const ledgerEventTime_1 = require("../retrieval/ledgerEventTime");
50
- const entityLookup_1 = require("../retrieval/entityLookup");
51
- const writer_1 = require("../memory/writer");
52
- const memoryMap_1 = require("../memorymap/memoryMap");
53
- const pipeline_1 = require("../indexer/pipeline");
54
- const walker_1 = require("../indexer/walker");
55
- const queryHelpers_1 = require("../retrieval/queryHelpers");
56
- const keyword_1 = require("../retrieval/keyword");
57
- const noteIdentity_1 = require("../retrieval/noteIdentity");
58
- const ledgerPresentation_1 = require("../retrieval/ledgerPresentation");
59
- const scope_1 = require("./scope");
60
- const lifecycle_1 = require("../memory/lifecycle");
61
- const inventory_1 = require("../memory/inventory");
62
- const readScope_1 = require("./readScope");
63
- const attach_1 = require("../evidence/attach");
64
- const options_1 = require("../evidence/options");
65
- const background_1 = require("../evidence/background");
66
- const contextRelevance_1 = require("./contextRelevance");
67
- const sanitize_1 = require("./sanitize");
68
- const digests_1 = require("../digests/digests");
69
- const worker_1 = require("../digests/worker");
70
- const log_1 = require("./log");
71
- const log = (0, log_1.createLogger)('engine');
72
- exports.DEFAULT_EMBED_BUDGETS = { writeMs: 5_000, readMs: 3_000 };
73
- /**
74
- * `RENOR_EMBED_WRITE_BUDGET_MS` and `RENOR_EMBED_READ_BUDGET_MS` set the two budgets; `0` for either (or both) lets
75
- * that kind of call wait as long as the provider's retries take, which is what every call did before they existed.
76
- */
77
- function resolveEmbedBudgets(env = process.env) {
78
- const read = (name, fallback) => {
79
- const raw = env[name]?.trim();
80
- if (raw === undefined || raw === '')
81
- return fallback;
82
- const value = Number(raw);
83
- if (!Number.isInteger(value) || value < 0 || (value > 0 && value < 100) || value > 600_000) {
84
- throw new Error(`${name} must be 0 (no limit) or a number of milliseconds between 100 and 600000`);
85
- }
86
- return value;
87
- };
88
- const writeMs = read('RENOR_EMBED_WRITE_BUDGET_MS', exports.DEFAULT_EMBED_BUDGETS.writeMs);
89
- const readMs = read('RENOR_EMBED_READ_BUDGET_MS', exports.DEFAULT_EMBED_BUDGETS.readMs);
90
- return writeMs === 0 && readMs === 0 ? undefined : { writeMs: writeMs || Infinity, readMs: readMs || Infinity };
91
- }
92
- /** `provider` as one call that has a budget sees it: every call it makes ends within `budgetMs` (see EmbedOptions). */
93
- function withBudget(provider, budgetMs) {
94
- if (budgetMs === undefined || !Number.isFinite(budgetMs))
95
- return provider;
96
- return {
97
- dimensions: provider.dimensions,
98
- identity: provider.identity,
99
- embed: texts => provider.embed(texts, { budgetMs }),
100
- embedQuery: text => provider.embedQuery ? provider.embedQuery(text, { budgetMs }) : provider.embed([text], { budgetMs }).then(vectors => vectors[0]),
101
- };
102
- }
103
- /** sqlite-vec table dimension, or null if the table does not exist. */
104
- function readEmbeddingTableDims(db) {
105
- const row = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'`).get();
106
- if (!row?.sql)
107
- return null;
108
- const m = row.sql.match(/FLOAT\[(\d+)\]/i);
109
- return m ? Number(m[1]) : null;
110
- }
111
- /**
112
- * Where a workspace keeps the vectors of the queries it has embedded: its own private state directory, the one
113
- * that holds the OAuth client registry (`MVP_AUTH_STATE_PATH`), which config validation keeps out of the memory
114
- * and history trees. So export, import and memory_status never see it, and it is one file per workspace.
115
- */
116
- function queryEmbeddingCachePath(config) {
117
- return path_1.default.join(path_1.default.dirname(config.authStatePath), 'query-embeddings.db');
118
- }
119
- /** Rows and vector bytes the read-path cache may hold: a few thousand distinct queries (3 KiB each at 768 dims). */
120
- exports.QUERY_CACHE_MAX_ENTRIES = 4096;
121
- exports.QUERY_CACHE_MAX_BYTES = 32 * 1024 * 1024;
122
- /**
123
- * The vector of a query already asked costs no second provider call. It is a cache: if it cannot be opened, the
124
- * file is derived and is removed once and started again, and if it still cannot be opened the provider is
125
- * returned as it is.
126
- */
127
- function withQueryCache(provider, config, scope) {
128
- const cachePath = queryEmbeddingCachePath(config);
129
- const open = () => new caching_1.CachingEmbeddingProvider(provider, {
130
- cachePath,
131
- serving: true,
132
- scope,
133
- maxEntries: exports.QUERY_CACHE_MAX_ENTRIES,
134
- maxBytes: exports.QUERY_CACHE_MAX_BYTES,
135
- });
136
- try {
137
- return open();
138
- }
139
- catch {
140
- try {
141
- for (const suffix of ['', '-wal', '-shm'])
142
- fs_1.default.rmSync(cachePath + suffix, { force: true });
143
- return open();
144
- }
145
- catch {
146
- return provider;
147
- }
148
- }
149
- }
150
- function createEmbeddingProvider(config,
151
- /** A test points the Vertex endpoint at a local stub and shortens its retry waits; production passes nothing. */
152
- vertexSeam,
153
- /**
154
- * The service keeps its connection to the embedding service across the pauses between two reads, up to three minutes
155
- * (src/net/poolFetch.ts). A bench or a command that builds the stack for one run leaves it off: it makes its requests
156
- * back to back and gains nothing from a pool that would outlive it, and it keeps the plain `fetch` it was measured with.
157
- */
158
- pooled = false) {
159
- if (config.embeddingProvider === 'none')
160
- return null;
161
- if (config.embeddingProvider === 'local') {
162
- return new local_1.LocalEmbeddingProvider(config.embeddingModel);
163
- }
164
- if (config.embeddingProvider === 'vertex') {
165
- // The provider's identity is its kind and model; the project and location it is served from are not in it. The
166
- // endpoint has no setting of its own (it is built from the location), so these two name it.
167
- const served = crypto_1.default.createHash('sha256').update(`${config.vertexProject ?? ''}\0${config.vertexLocation ?? ''}`).digest('hex').slice(0, 12);
168
- // The query cache is outermost: a query already asked is answered from it with the service down, and only a query
169
- // that must go to the service meets the interactive budget and the failure memory in front of it.
170
- // Document embeddings from concurrent index passes share requests (coalescing.ts): below the budget, so each caller's
171
- // budget still ends its own call, and above the service, so a request holds at most the usual four in flight.
172
- return withQueryCache(new interactive_1.InteractiveEmbeddingProvider(new coalescing_1.CoalescingEmbeddingProvider(new vertex_1.VertexEmbeddingProvider({
173
- keyFile: config.vertexKeyFile,
174
- project: config.vertexProject,
175
- location: config.vertexLocation,
176
- model: config.embeddingModel,
177
- dimensions: config.embeddingDims,
178
- ...vertexSeam,
179
- // An embedding is a pure function of its text, so a request whose idle connection the service closed meanwhile is sent again.
180
- // A connection it has to open for a read is told to that read's log line (callPhases.ts `connect`).
181
- ...(pooled ? { request: (0, poolFetch_1.poolFetch)({ retryOnReset: true, onConnect: ms => (0, callPhases_1.recordPhase)('connect', ms) }) } : {}),
182
- }))), config, `p-${served}`);
183
- }
184
- const keys = (config.embeddingApiKey ?? '').split(',').map(k => k.trim()).filter(Boolean);
185
- return new openai_1.OpenAIEmbeddingProvider(keys, config.embeddingModel, config.embeddingBaseURL, config.embeddingDims);
186
- }
187
- function attachEngine(db, config, override) {
188
- const provider = override === undefined ? createEmbeddingProvider(config, undefined, true) : override;
189
- if (!provider) {
190
- // What this process saves gets no vector: an index that has a model's vectors is marked unfinished first.
191
- if ((0, vectorIndex_1.markVectorsUnfinished)(db)) {
192
- log.info('embeddings_unconfigured', { detail: 'no embedding provider is configured: what is saved now gets no vector, so the index is marked unfinished; the next start with its model embeds those chunks before semantic search reads the index again' });
193
- }
194
- return { provider: null, vectorIndex: null, dims: readEmbeddingTableDims(db) };
195
- }
196
- const existing = readEmbeddingTableDims(db);
197
- if (existing !== null && existing !== provider.dimensions) {
198
- throw new Error(`Embedding dimension mismatch: index.db chunk_embeddings is FLOAT[${existing}] but the configured provider is ${provider.dimensions}-d. `
199
- + `Do not mix Voyage/e5/other models in one book (ADR-0003). Set MVP_EMBEDDING_DIMS and MVP_EMBEDDING_MODEL to match the index, or use a separate MVP_DB_PATH.`);
200
- }
201
- const vectorIndex = new vectorIndex_1.VectorIndex(db, provider.dimensions);
202
- vectorIndex.initTable();
203
- claimEmbeddingSpace(db, vectorIndex, provider);
204
- const modelKey = override === undefined
205
- ? (0, contextRelevance_1.embeddingModelKey)(config.embeddingProvider, config.embeddingModel, config.embeddingDims)
206
- : injectedModelKey(provider);
207
- const embedBudgets = resolveEmbedBudgets();
208
- return {
209
- provider,
210
- vectorIndex,
211
- dims: provider.dimensions,
212
- ...(modelKey ? { modelKey } : {}),
213
- ...(embedBudgets ? { embedBudgets } : {}),
214
- };
215
- }
216
- /**
217
- * The model key of an injected provider, from the identity it declares and the size it embeds at: exactly what the
218
- * configured path composes, so a caller that brings its own provider (a bench, a test, an in-process harness) keeps
219
- * the calibrated relevance bar its model has instead of losing the verdict altogether. An injected provider that
220
- * declares no identity names no model, so it has no bar: `undefined`, and the signal stays silent rather than
221
- * guessing (the eval-hash stub is exactly this case — lexical vectors, and nothing calibrated to compare them to).
222
- */
223
- function injectedModelKey(provider) {
224
- const identity = provider.identity;
225
- return identity ? (0, contextRelevance_1.embeddingModelKey)(identity.kind, identity.model, provider.dimensions) : undefined;
226
- }
227
- /**
228
- * Dimensions alone cannot tell two embedding models apart, so the index records the space its vectors are in
229
- * (provider, model, dimensions). A new index takes the configured model's space. An index whose vectors are in
230
- * another space, or in none anyone can vouch for (it holds vectors and records none: it was built before spaces
231
- * were recorded), is not named and is not read: search would compare numbers from different models. It is rebuilt
232
- * instead (`engineRebuildEmbeddings`, which the service starts), and until that finishes semantic search is off
233
- * (`semanticReady`). An index that holds chunks and no vector at all was written without an embedding provider (a
234
- * workspace that searched by keyword until now): it is named and marked unfinished, so the same background job
235
- * embeds every one of its chunks, once, before semantic search reads it. A provider that declares no identity claims
236
- * nothing, and nothing is held back for it.
237
- */
238
- function claimEmbeddingSpace(db, vectorIndex, provider) {
239
- const wanted = (0, space_1.embeddingSpace)(provider);
240
- if (!wanted)
241
- return;
242
- const stored = vectorIndex.space();
243
- if (stored === wanted)
244
- return;
245
- if (stored === null && !vectorIndex.hasVectors()) {
246
- const chunks = db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
247
- if (chunks === 0) {
248
- vectorIndex.setSpace(wanted); // a new index: nothing to embed, so it is whole
249
- return;
250
- }
251
- vectorIndex.beginRebuild(wanted); // there is no vector to clear: this names the index and marks it unfinished
252
- log.info('embeddings_backfill', { chunks, model: wanted, detail: 'index.db holds no vectors (it was written without an embedding provider): every chunk is embedded in the background; until then semantic search is off and keyword search keeps working' });
253
- return;
254
- }
255
- log.warn('embeddings_model_mismatch', {
256
- stored: stored ?? 'unrecorded',
257
- model: wanted,
258
- detail: 'search cannot use vectors from another model: a rebuild embeds every chunk again, and until it finishes semantic search is off and keyword search keeps working',
259
- });
260
- }
261
- /**
262
- * Whether a semantic search may read this engine's index: every vector in it is in the configured model's space and
263
- * none is still to come. An index in another space, in none anyone can vouch for, or part way through a rebuild is
264
- * not read: search would compare numbers from different models, or find only part of the index. A provider that
265
- * declares no identity makes no claim, so it is never held back.
266
- */
267
- function semanticReady(engine) {
268
- if (!engine.provider || !engine.vectorIndex)
269
- return false;
270
- const wanted = (0, space_1.embeddingSpace)(engine.provider);
271
- if (!wanted)
272
- return true;
273
- return engine.vectorIndex.space() === wanted && !engine.vectorIndex.rebuilding();
274
- }
275
- /** What a caller is told while semantic search is off for a rebuild. */
276
- function rebuildNotice(db, engine) {
277
- const chunks = db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
278
- const left = engine.vectorIndex.space() === (0, space_1.embeddingSpace)(engine.provider) ? (0, rebuild_1.chunksWithoutVectors)(db) : chunks;
279
- return `Semantic search is off while this workspace's embeddings are built with the configured model (${left} of ${chunks} chunks left). Keyword search works meanwhile.`;
280
- }
281
- /** Why something failed, on one line a client may be shown (host paths redacted). */
282
- function failureReason(err, max) {
283
- return (0, sanitize_1.sanitizeErrorMessage)(err).replace(/\s+/g, ' ').trim().slice(0, max);
284
- }
285
- /**
286
- * The last search by meaning that failed, per engine, until one succeeds: memory_status reports it, so a workspace whose
287
- * embedding service stopped answering is not reported healthy (it makes no embeddings call of its own).
288
- */
289
- const semanticFailures = new WeakMap();
290
- /** Whether keyword search can run: its index table exists. A damaged or half-migrated index file has none. */
291
- function keywordIndexReady(db) {
292
- return (0, statements_1.prepareOnce)(db, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_fts'`).get() !== undefined;
293
- }
294
- /** What a caller is told when a search by meaning failed: why, and what still answers. */
295
- function semanticFailureNotice(err) {
296
- return `Semantic search is unavailable right now (${failureReason(err, 200)}), so nothing here was found by meaning; `
297
- + 'the saved facts and keyword search do not need it and still answer. The next request tries semantic search again.';
298
- }
299
- const embeddingRebuilds = new WeakMap();
300
- /**
301
- * Rebuild, in the background, every vector this engine's index holds with the configured model, when its index is
302
- * not whole in that model's space (see `semanticReady`), and record the space once it is. The clearing and naming
303
- * happen before this returns; the embedding happens after, a batch at a time. The index is whole only when every
304
- * chunk has a vector. A provider that is down, or a chunk it will not embed, interrupts the rebuild; it is resumed
305
- * after a wait that grows (30 s, doubling, up to 5 min), semantic search stays off, and the log says how many chunks
306
- * are left and which one was refused. So it ends only when the index is whole or `abort` is called. Returns null
307
- * when there is nothing to do; a rebuild already running for this index is returned rather than started twice.
308
- */
309
- function engineRebuildEmbeddings(db, engine, options = {}) {
310
- const { provider, vectorIndex } = engine;
311
- if (!provider || !vectorIndex)
312
- return null;
313
- const space = (0, space_1.embeddingSpace)(provider);
314
- if (!space || semanticReady(engine))
315
- return null;
316
- const running = embeddingRebuilds.get(vectorIndex);
317
- if (running)
318
- return running;
319
- const { outageDelayMs = (outage) => Math.min(300_000, 30_000 * 2 ** outage), ...rebuildOptions } = options;
320
- const controller = new AbortController();
321
- const startedAt = Date.now();
322
- let waiting = false;
323
- const wait = async (ms) => {
324
- waiting = true;
325
- try {
326
- await (0, rebuild_1.sleep)(ms, controller.signal);
327
- }
328
- finally {
329
- waiting = false;
330
- }
331
- };
332
- const run = async () => {
333
- const chunks = () => db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
334
- log.info('embedding_rebuild_started', { chunks: chunks(), model: space, detail: 'semantic search is off until it finishes' });
335
- for (let outage = 0; !controller.signal.aborted; outage++) {
336
- try {
337
- const result = await (0, rebuild_1.rebuildEmbeddings)(db, vectorIndex, provider, space, { ...rebuildOptions, signal: controller.signal });
338
- if (result.done) {
339
- log.info('embedding_rebuild_finished', { ms: Date.now() - startedAt, detail: 'semantic search is on' });
340
- return;
341
- }
342
- }
343
- catch (err) {
344
- const delay = outageDelayMs(outage);
345
- log.error('embedding_rebuild_interrupted', { unembedded: (0, rebuild_1.chunksWithoutVectors)(db), chunks: chunks(), retryInS: Math.round(delay / 1000), detail: 'semantic search stays off' }, err);
346
- await wait(delay);
347
- }
348
- }
349
- };
350
- // The clearing and naming run now, in the synchronous start of the first call, so the index is never left
351
- // looking whole while it holds another model's vectors.
352
- const rebuild = {
353
- abort: () => controller.abort(),
354
- waiting: () => waiting,
355
- done: run().catch((err) => log.error('embedding_rebuild_failed', {}, err)).finally(() => embeddingRebuilds.delete(vectorIndex)),
356
- };
357
- embeddingRebuilds.set(vectorIndex, rebuild);
358
- return rebuild;
359
- }
360
- /** The rebuild `engineRebuildEmbeddings` started for this engine's index that has not ended yet, or null; starts none. */
361
- function runningEmbeddingRebuild(engine) {
362
- return (engine.vectorIndex && embeddingRebuilds.get(engine.vectorIndex)) ?? null;
363
- }
364
- /** A search shows the path of every hit it judges, and the notes it finds are the same ones read after read. */
365
- const clientPaths = new Map();
366
- const MAX_CLIENT_PATHS = 4096;
367
- function clientPath(memoryDir, stored) {
368
- // A relative path resolves against the working directory, so it is part of the key then, and only then (as in
369
- // path.resolve, which asks for it only when it has to).
370
- const key = path_1.default.isAbsolute(stored) && path_1.default.isAbsolute(memoryDir) ? `${memoryDir}\0${stored}` : `${process.cwd()}\0${memoryDir}\0${stored}`;
371
- let shown = clientPaths.get(key);
372
- if (shown === undefined) {
373
- const resolved = path_1.default.resolve(stored);
374
- const root = path_1.default.resolve(memoryDir);
375
- shown = resolved === root || resolved.startsWith(root + path_1.default.sep) ? path_1.default.relative(root, resolved) || stored : path_1.default.basename(stored);
376
- if (clientPaths.size >= MAX_CLIENT_PATHS)
377
- clientPaths.clear();
378
- clientPaths.set(key, shown);
379
- }
380
- return shown;
381
- }
382
- function resolveEngineEvidence(config, env = process.env, idle = null) {
383
- if (!(0, attach_1.searchEvidenceEnabled)(env))
384
- return undefined;
385
- const options = (0, options_1.resolveEvidenceToolOptions)({ dbPath: config.dbPath }, env);
386
- // Bounded-wait evidence (RENOR_EVIDENCE_ASYNC, default off) is read here and nowhere else: a background extraction
387
- // needs a long-lived server to finish in, and the CLI, the benchmark's entry point, must always wait for its table.
388
- const background = (0, background_1.resolveEvidenceBackground)(env);
389
- if (background && idle) {
390
- background.hold = flight => {
391
- idle.track(() => flight).catch(() => undefined); // its failure is the flight's to report
392
- };
393
- }
394
- return { options: background ? { ...options, background } : options, extractor: (0, options_1.createEvidenceExtractor)(options) };
395
- }
396
- function resolveEngineDigests(db, config, engine, idle = null, env = process.env) {
397
- const options = (0, digests_1.resolveDigestOptions)({ dbPath: config.dbPath }, env);
398
- if (!options)
399
- return null;
400
- if (!engine.provider) {
401
- log.info('digests_off', { detail: 'RENOR_DIGESTS is on, but this workspace has no embedding provider, so digests are off for it' });
402
- return null;
403
- }
404
- const worker = new worker_1.DigestWorker({
405
- db,
406
- config: { memoryDir: config.memoryDir, ledgerRelPath: config.ledgerRelPath },
407
- options,
408
- provider: engine.provider,
409
- track: idle ? work => idle.track(work) : undefined,
410
- });
411
- log.info('digests_on', { model: options.model, maxUsdPerDay: options.maxUsdPerDay });
412
- return { options, worker };
413
- }
414
- /**
415
- * What search_memory answers on a workspace without semantic search. It searched nothing, and the error code
416
- * alone left the agent without a next step, so `note` names the search every workspace has.
417
- */
418
- const SEMANTIC_UNAVAILABLE_NOTE = 'This workspace has no semantic search, so nothing was searched by meaning. Use search_text instead: it matches exact words, so search one distinctive word (a name, a project, a term).';
419
- function semanticUnavailable() {
420
- return { ok: false, error: 'semantic-unavailable', note: SEMANTIC_UNAVAILABLE_NOTE, hits: [] };
421
- }
422
- async function engineSearchMemory(db, engine, memoryDir, query, topK, scope, evidence, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL,
423
- /** Out-param for get_context: the query embedding and the best raw similarity among the returned (scoped, live) semantic hits. */
424
- probe,
425
- /** Write-time digests (RENOR_DIGESTS): when present, the digests of the window's notes lead the hits as one table. No model call. */
426
- digests,
427
- /** Per-read default: get_context passes 0; searches leave this unset for the default cap of 2. */
428
- noteCapWhenUnset,
429
- /** Internal floor input: expose note similarities so the MCP search can filter its completed page. */
430
- showSimilarity = false,
431
- /** search_memory's reader layout (readerLayout.ts) needs each chunk's place in its note: the ids ride on the hits until it has used them. */
432
- withChunkIds = false,
433
- /** search_memory under RENOR_SEARCH_BEST_FIRST: the window best match first (retrieval/orchestrator.ts). */
434
- bestFirst = false) {
435
- // A query with no searchable characters can never truthfully match stored data. Refuse before
436
- // any embedding call (eval-hash or provider) so whitespace-only input yields no hits instead
437
- // of an arbitrary semantic match, matching search_history's blank-query handling.
438
- if ((0, invisibleText_1.isBlankOrInvisible)(query)) {
439
- return { ok: true, hits: [] };
440
- }
441
- if (!engine.provider || !engine.vectorIndex) {
442
- return semanticUnavailable();
443
- }
444
- // Never search a mixed or unfinished space: the caller falls back to keyword search and is told why.
445
- if (!semanticReady(engine)) {
446
- return { ok: false, error: 'semantic-rebuilding', notice: rebuildNotice(db, engine), hits: [] };
447
- }
448
- const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath });
449
- // The query is embedded within the read budget: past it the search says so (below) and the caller answers from
450
- // the facts and keyword search, as it does for any failure of the provider.
451
- const provider = withBudget(engine.provider, engine.embedBudgets?.readMs);
452
- const vectorIndex = engine.vectorIndex;
453
- const asked = probe ?? {};
454
- const scopeNorm = (0, lifecycle_1.scopeFilterFor)(scope);
455
- const inScope = scopeNorm ? (0, readScope_1.createReadScope)(db, scopeNorm) : null;
456
- // The scope is judged before the window is cut, so a scope whose memories rank below other scopes' still gets its
457
- // best ones (memory scout, finding 7: judged only after the cut, they were all gone); and again on what comes back.
458
- // Which chunks the scope holds is read from every chunk of the index (and the scope of every ledger event), so it is
459
- // read once and kept until something is written; reads that follow one another, as an agent's do, share it.
460
- const scoped = inScope
461
- ? {
462
- shows: h => inScope.has(h, clientPath(memoryDir, h.notePath)),
463
- chunkIds: () => (0, readCache_1.readUntilWrite)(db, `scopedChunkIds\0${scopeNorm}\0${memoryDir}`, () => inScope.chunkIds(stored => clientPath(memoryDir, stored))),
464
- }
465
- : undefined;
466
- const search = () => (0, orchestrator_1.retrieveContext)(db, vectorIndex, provider, query, topK, [], {}, { includeHistory, isLedgerNote: isLedger, probe: asked, scope: scoped, noteCapWhenUnset, ...(bestFirst ? { bestFirst } : {}) });
467
- let result;
468
- try {
469
- result = await search();
470
- // What a save could not embed is embedded now, once the provider answers, and searched with the rest.
471
- if (await finishUnfinishedFiles(db, memoryDir, engine, provider, query, asked))
472
- result = await search();
473
- }
474
- catch (err) {
475
- // Search by meaning failed: the embedding provider is down, or refuses the key or the quota (or the semantic lane
476
- // broke, which the log shows). The facts and keyword search do not need it, so a caller answers with those and
477
- // says why nothing was found by meaning, instead of failing whole.
478
- log.error('semantic_search_failed', {}, err);
479
- semanticFailures.set(engine, { at: new Date().toISOString(), reason: failureReason(err, 120) });
480
- return { ok: false, error: 'semantic-unavailable', notice: semanticFailureNotice(err), hits: [] };
481
- }
482
- semanticFailures.delete(engine);
483
- // Scope must be enforced before any full-note hydration/evidence extraction.
484
- // Otherwise a scoped query could derive evidence from notes outside its scope
485
- // even if those ranked hits are filtered from the final response later.
486
- const scopedRetrievalHits = inScope
487
- ? inScope.hits(result.hits, h => clientPath(memoryDir, h.notePath))
488
- : result.hits;
489
- if (probe) {
490
- let top;
491
- for (const h of scopedRetrievalHits)
492
- if (typeof h.similarity === 'number' && (top === undefined || h.similarity > top.similarity))
493
- top = h;
494
- probe.topSimilarity = top ? top.similarity : null;
495
- // The same hit as the hits below show it: a ledger event as what it says (presentLedgerEvents judges each hit alone).
496
- const shown = top && (0, ledgerPresentation_1.presentLedgerEvents)([top], isLedger)[0];
497
- probe.topHit = shown ? { notePath: clientPath(memoryDir, shown.notePath), headingPath: shown.headingPath, text: shown.text } : undefined;
498
- // Every returned semantic hit as the memories show it, with its raw cosine: a budget reads which of them are above the bar.
499
- probe.similarities = scopedRetrievalHits.flatMap(h => {
500
- const asShown = typeof h.similarity === 'number' ? (0, ledgerPresentation_1.presentLedgerEvents)([h], isLedger)[0] : undefined;
501
- return asShown ? [{ notePath: clientPath(memoryDir, asShown.notePath), headingPath: asShown.headingPath, text: asShown.text, similarity: h.similarity }] : [];
502
- });
503
- }
504
- const closedEvents = (0, ledgerLiveness_1.closedLedgerEvents)(db);
505
- const noteDate = (0, orchestrator_1.createNoteDater)(db);
506
- // Without an evidence model no second pass runs, and nothing is said about one: on every search of such a workspace
507
- // (every hosted one) a report that nothing was missing told the agent nothing. With a model, every state is reported,
508
- // and so is the kill switch ("off").
509
- const attached = evidence?.extractor
510
- ? await (0, attach_1.attachSearchEvidence)({
511
- // The ledger note holds every scope's events, so hydration drops other scopes' before closed ones.
512
- transformBody: text => (0, ledgerLiveness_1.resolveLedgerHits)([{ text: inScope ? inScope.body(text) : text, reason: 'evidence-source' }], closedEvents, includeHistory)[0]?.text ?? '',
513
- query,
514
- // Evidence hydrates whole notes and prints one "Session dated" line above each: the note's date. A hit's own
515
- // date is its excerpt's, and one ledger event's would date every other event in the same body as well.
516
- hits: scopedRetrievalHits.map(h => ({ ...h, session_date: noteDate(h.notePath) })),
517
- vaultPath: memoryDir,
518
- options: evidence.options,
519
- extractor: evidence.extractor,
520
- enabled: true,
521
- })
522
- : evidence
523
- ? null
524
- : { hits: scopedRetrievalHits, evidence: { status: 'skipped', reason: 'flag-off' } };
525
- const shown = (0, ledgerPresentation_1.presentLedgerEvents)(scopedRetrievalHits, isLedger);
526
- // The evidence table, when there is one, leads the hits the agent is shown; they keep their own dates.
527
- let led = attached ? (0, attach_1.prependEvidenceHit)(shown, attached.evidence) : shown;
528
- // The digest table follows it. Only notes of the window the scope already allowed are read, so a scoped read never sees
529
- // another scope's digests, and no model is called: it reads the digest cache and the query embedding the search made.
530
- if (digests) {
531
- try {
532
- const table = await (0, digests_1.digestTable)({
533
- db, query, hits: scopedRetrievalHits, options: digests.options, provider, queryVector: asked.queryVector, isLedgerNote: isLedger,
534
- });
535
- if (table.items.length > 0) {
536
- const lead = (0, digests_1.leadingSummaryHits)(led);
537
- led = [...led.slice(0, lead), (0, digests_1.digestItemsToHit)(table.items), ...led.slice(lead)];
538
- }
539
- }
540
- catch (err) {
541
- // Optional augmentation: a fault here never fails the search.
542
- log.error('digests_read_failed', {}, err);
543
- }
544
- }
545
- const isConversation = withChunkIds ? (0, readerLayout_1.conversationNoteMatcher)(db) : () => false;
546
- const hits = led.map(h => ({
547
- notePath: clientPath(memoryDir, h.notePath),
548
- headingPath: h.headingPath,
549
- text: h.text,
550
- score: h.score,
551
- reason: h.reason,
552
- ...(h.session_date ? { session_date: h.session_date } : {}),
553
- // Only a conversation's chunks: the layout merges those and no others.
554
- ...(withChunkIds && 'chunkId' in h && typeof h.chunkId === 'number' && h.chunkId > 0 && isConversation(h.notePath) ? { chunkId: h.chunkId } : {}),
555
- // The author of a fact an identity question read: the reader's view marks it (mvp/factMarkers.ts).
556
- ...('author' in h && h.author !== undefined ? { author: h.author } : {}),
557
- // Raw cosine before boosts, rounded as in the checked floor arm. A ledger event is no note:
558
- // its chunk is event markup, and its cosine says little about what it holds (contextRelevance.ts).
559
- ...(showSimilarity && typeof h.similarity === 'number' && !isLedger(h.notePath) ? { similarity: Math.round(h.similarity * 1000) / 1000 } : {}),
560
- }));
561
- return {
562
- ok: true,
563
- rankedBy: result.rankedBy,
564
- hits,
565
- // Present whenever an evidence model is configured, and when the kill switch is off: `state` is how a caller tells
566
- // an intended "off" from a pass that failed or did not apply.
567
- ...(attached ? { evidence: (0, attach_1.productEvidenceReport)(attached.evidence) } : {}),
568
- ...(scopeNorm ? { scope: scopeNorm } : {}),
569
- };
570
- }
571
- function engineSearchText(db, memoryDir, query, topK, scope, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL, ledgerRecords, matchOverride) {
572
- const fts = (0, statements_1.prepareOnce)(db, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_fts'`).get();
573
- if (!fts)
574
- return { ok: false, error: 'fts-unavailable', results: [] };
575
- // matchOverride (H11b): an FTS expression built by the caller (an OR over the question's content words); absent, the whole query as one phrase.
576
- const match = matchOverride ?? (0, queryHelpers_1.quoteFtsQuery)(query);
577
- if (!match)
578
- return { ok: true, results: [] };
579
- const scopeNorm = (0, lifecycle_1.scopeFilterFor)(scope);
580
- const inScope = scopeNorm ? (0, readScope_1.createReadScope)(db, scopeNorm) : null;
581
- const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath });
582
- // A search may judge thousands of rows of one note (the ledger is one note), so each stored path is made the path a
583
- // client is shown once per search, not once per row.
584
- const shownPaths = new Map();
585
- const shownPath = (stored) => {
586
- let shown = shownPaths.get(stored);
587
- if (shown === undefined)
588
- shownPaths.set(stored, (shown = clientPath(memoryDir, stored)));
589
- return shown;
590
- };
591
- const toResult = (r) => ({
592
- notePath: shownPath(r.notePath),
593
- headingPath: r.headingPath,
594
- text: r.text,
595
- score: r.score,
596
- reason: r.reason,
597
- });
598
- const closedEvents = (0, ledgerLiveness_1.closedLedgerEvents)(db);
599
- let prose;
600
- try {
601
- // A row is shown when its event is live (liveness is resolved first), it is not the ledger's own markup, and the
602
- // scope read holds it. All three are judged before the page is cut, from a read that goes on until the page is full
603
- // or the index has no more rows: a closed event, a record of the ledger or another scope's row never takes a place
604
- // from a row that is shown, however many of them rank above it. The first read is the window of the best
605
- // `topK * 10` rows (at most 200) this search has always read.
606
- prose = (0, queryHelpers_1.shownPage)(n => (0, keyword_1.keywordSearch)(db, match, n), rows => {
607
- const live = (0, ledgerPresentation_1.keywordLedgerHits)((0, ledgerLiveness_1.resolveLedgerHits)(rows.map(r => toResult({ ...r, score: Math.abs(r.score), reason: 'fts' })), closedEvents, includeHistory), isLedger, query, ledgerRecords);
608
- return inScope ? inScope.hits(live, r => r.notePath) : live;
609
- }, topK, Math.min(topK * 10, 200));
610
- }
611
- catch (err) {
612
- const message = err instanceof Error ? err.message : String(err);
613
- return { ok: false, error: 'fts-query', detail: message.slice(0, 200), results: [] };
614
- }
615
- // Only the prose hits actually returned count as "already have"; scope is applied
616
- // before the identity limit.
617
- const have = new Set(prose.map(h => h.notePath));
618
- // The ledger is a note too (`ledger.md` names it), so its opening chunk obeys the same liveness and scope rules:
619
- // judged by the scope of the event it opens with, not by the ledger's path, and before the identity limit.
620
- const named = (0, ledgerLiveness_1.resolveLedgerHits)((0, noteIdentity_1.selectNoteIdentityHits)(db, query, noteIdentity_1.NOTE_IDENTITY_EXTRA, new Set(), (p, opening) => {
621
- const client = clientPath(memoryDir, p);
622
- return !have.has(client) && !(0, ledgerPresentation_1.isLedgerPreamble)(opening, isLedger) && (!inScope || inScope.has(opening, client));
623
- }).map(toResult), closedEvents, includeHistory);
624
- // An event is dated by when it was saved, as search_memory dates it: its time was in the record's heading alone, which
625
- // a client is not shown (ledgerPresentation.ts). Read from the stored record, before it is shown as the user's words,
626
- // which may hold a line that looks like a record's heading.
627
- const eventTime = (0, ledgerEventTime_1.createLedgerEventClock)(db);
628
- const boundTime = (0, orchestrator_1.createNoteTimeBound)(db);
629
- const results = (0, ledgerPresentation_1.presentLedgerEvents)([...prose, ...named].map(r => {
630
- const derived = isLedger(r.notePath) ? eventTime(r) : undefined;
631
- if (derived === undefined)
632
- return r;
633
- const session_date = boundTime(path_1.default.resolve(memoryDir, r.notePath), derived);
634
- return { ...r, session_date };
635
- }), isLedger, ledgerRecords);
636
- return {
637
- ok: true,
638
- results,
639
- ...(scopeNorm ? { scope: scopeNorm } : {}),
640
- };
641
- }
642
- /**
643
- * What lookup_entity actually searches — all four mechanisms, in order, because
644
- * a narrower description would be a false contract: a term can resolve on a
645
- * substring of a canonical name or on a substring of a source note path, and
646
- * none of them see words inside note text.
647
- */
648
- exports.ENTITY_LOOKUP_SURFACE = 'registered entities, matched in order by exact canonical name, frontmatter alias, substring of a canonical name, or substring of a source note path — never words inside note text';
649
- function engineLookupEntity(db, vault, name, scope) {
650
- const scopeNorm = scope === undefined ? undefined : (0, scope_1.normalizeScope)(scope);
651
- // The scope constrains the SEARCH, not just the answer: the same restriction
652
- // narrows the candidate queries and the count, so a scoped lookup cannot
653
- // distinguish "no such entity" from "exists in another scope" — the two must
654
- // be byte-identical responses or the miss reason is a cross-scope oracle.
655
- // A note's entity is judged by its note's path, a remembered fact's subject by
656
- // the scopes its facts were saved in (entityScopeFilter), and with no scope while it has a fact (liveEntityFilter): a
657
- // subject whose every fact is retired is named nowhere.
658
- const filter = scopeNorm === undefined ? (0, entityLookup_1.liveEntityFilter)() : (0, entityLookup_1.entityScopeFilter)(scopeNorm);
659
- const row = (0, entityLookup_1.lookupEntity)(db, name, filter);
660
- const miss = () => {
661
- return {
662
- ok: true,
663
- entity: null,
664
- reason: 'no-entity-match',
665
- searched: exports.ENTITY_LOOKUP_SURFACE,
666
- // Scoped to the same restriction as the lookup: never a workspace-wide
667
- // count handed to a caller that may only read one scope. The same count
668
- // memory_status reports, so the two never name different numbers.
669
- entitiesIndexed: (0, inventory_1.countEntities)(db, vault, filter),
670
- note: `Nothing in ${scopeNorm ?? 'this workspace'} is registered as an entity matching "${name}". lookup_entity searches ${exports.ENTITY_LOOKUP_SURFACE}, so a word that appears only inside saved note text (a daily note, a saved fact body) is not an entity. Use search_text for that word, or search_memory for meaning where the workspace has semantic search.`,
671
- ...(scopeNorm === undefined ? {} : { scope: scopeNorm }),
672
- };
673
- };
674
- if (!row)
675
- return miss();
676
- const sourceNote = clientPath(vault.memoryDir, row.sourceNote);
677
- if (scopeNorm !== undefined) {
678
- return {
679
- ok: true,
680
- entity: { canonicalName: row.canonicalName, type: row.type, aliases: row.aliases, sourceNote },
681
- ...(scopeNorm === 'global' ? {} : { scope: scopeNorm }),
682
- };
683
- }
684
- return {
685
- ok: true,
686
- entity: {
687
- canonicalName: row.canonicalName,
688
- type: row.type,
689
- aliases: row.aliases,
690
- sourceNote,
691
- },
692
- };
693
- }
694
- function engineStatus(db, engine, vault) {
695
- const counts = (0, inventory_1.countInventory)(db, vault);
696
- // What cannot answer right now, said rather than left for the agent to infer from empty results. Absent when healthy.
697
- const failed = semanticFailures.get(engine);
698
- const degraded = {
699
- ...(keywordIndexReady(db) ? {} : { keyword: 'keyword search cannot run: its index is missing' }),
700
- ...(failed ? { semantic: `the last search by meaning failed at ${failed.at}: ${failed.reason}` } : {}),
701
- };
702
- return {
703
- ...counts,
704
- embeddingDims: engine.dims,
705
- // Whether semantic search can answer now: false with no provider, and false until the embeddings are whole.
706
- semantic: semanticReady(engine),
707
- ...(engine.provider && engine.vectorIndex && !semanticReady(engine)
708
- ? { embeddingRebuild: { chunksLeft: engine.vectorIndex.space() === (0, space_1.embeddingSpace)(engine.provider) ? (0, rebuild_1.chunksWithoutVectors)(db) : counts.chunks, chunks: counts.chunks } }
709
- : {}),
710
- // With an embedding provider, the chunks search by meaning cannot find, because they have no vector: a save whose
711
- // embedding failed (the next save, or a search, embeds them), notes an import wrote (the background job embeds
712
- // them), or a rebuild that is not done.
713
- ...(engine.vectorIndex ? { chunksWithoutVectors: (0, rebuild_1.chunksWithoutVectors)(db) } : {}),
714
- ...(Object.keys(degraded).length > 0 ? { degraded } : {}),
715
- };
716
- }
717
- /** `interactive`: an agent is waiting on this pass, so embedding is held to the write budget (see EmbedBudgets). */
718
- function indexOptionsFor(engine, interactive = false) {
719
- return engine.provider && engine.vectorIndex
720
- ? {
721
- embeddingProvider: engine.provider,
722
- vectorIndex: engine.vectorIndex,
723
- ...(interactive && engine.embedBudgets && Number.isFinite(engine.embedBudgets.writeMs) ? { embedBudgetMs: engine.embedBudgets.writeMs } : {}),
724
- }
725
- : {};
726
- }
727
- /**
728
- * Files an index pass has in flight at once. A file holds its slot while its embedding request is out, so the slot count is
729
- * how many requests an index pass can batch: with the coalescing provider (coalescing.ts) the requests stay at four in
730
- * flight whatever this is. Value set by the measurements in engine/docs/perf (see the note there).
731
- */
732
- exports.INDEX_FILE_CONCURRENCY = 48;
733
- /**
734
- * `interactive`: an agent waits on this pass (the `index_memory` tool), so a file whose embedding the service will not give
735
- * within the write budget is left unfinished and the pass returns; a background job then embeds it
736
- * (`engineFinishIndexInBackground`), however long the service takes. Unset: the provider waits as long as its own retries take.
737
- */
738
- async function engineIndex(db, config, engine, pass = {}) {
739
- const stats = await (0, pipeline_1.indexVault)(db, config.memoryDir, indexOptionsFor(engine, pass.interactive === true), exports.INDEX_FILE_CONCURRENCY);
740
- if (stats.errors > 0 && pass.interactive)
741
- engineFinishIndexInBackground(db, config, engine);
742
- return stats;
743
- }
744
- /**
745
- * What this process last reconciled each file with the index at: the file's stamp when `indexFile` last
746
- * succeeded on it. Kept per database, since another index has reconciled nothing.
747
- */
748
- const reconciledStamps = new WeakMap();
749
- /** Per database: the stamp each note over the size cap had when it was last named, so it is said once per change and not on every save. */
750
- const tooLargeStamps = new WeakMap();
751
- /**
752
- * Per database: the files a pass left unsettled, whose stamp this process does not record as reconciled (a file changed
753
- * within a clock tick of its scan, or whose pass failed or was not reached). A save checks them again, as a full scan
754
- * would, because it cannot tell from the files it wrote that they are done.
755
- */
756
- const unsettledFiles = new WeakMap();
757
- /**
758
- * Per database: when this process last checked every file of the vault (a full pass). Until one has run, a save walks the
759
- * vault (the first pass in a process); after that a save walks it again once FULL_VAULT_CHECK_MS has passed.
760
- */
761
- const walkedAt = new WeakMap();
762
- /**
763
- * How long note saves check only the files they wrote before every file of the vault is checked again. An edit made
764
- * outside the engine (in Obsidian, say) is therefore seen by a save once this long has passed since the last full check;
765
- * index_memory sees it at once.
766
- */
767
- exports.FULL_VAULT_CHECK_MS = 30_000;
768
- /** Every stat field a change shows in. The change time is what catches a rewrite that puts the modification time back. */
769
- const stampOf = (f) => `${f.mtimeMs}:${f.ctimeMs}:${f.size}:${f.ino}`;
770
- /**
771
- * A timestamp is only as fine as the file system's clock tick (two seconds on the coarsest), so a file written,
772
- * scanned and written again within one tick, to the same size, carries the same stamp both times; the same goes
773
- * for a rewrite that puts the modification time back, within a tick of the first change. A file whose
774
- * modification or change time is this close to a scan is therefore not trusted by its stamp: the next scan
775
- * reads it again.
776
- */
777
- exports.TIMESTAMP_TICK_MS = 2_000;
778
- const stampIsTrusted = (f, scanStart) => Math.min(Math.abs(scanStart - f.mtimeMs), Math.abs(scanStart - f.ctimeMs)) >= exports.TIMESTAMP_TICK_MS; // false for a failed stat (NaN)
779
- /** Run `work` over `items`, `limit` at a time. */
780
- async function inPool(items, limit, work) {
781
- let next = 0;
782
- await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
783
- while (next < items.length)
784
- await work(items[next++]);
785
- }));
786
- }
787
- /**
788
- * Bring the index up to date with the vault as a full `indexVault` would, reading only what changed. A file goes
789
- * through the same `indexFile` a full pass runs when the index has no finished row for it, or its stamp
790
- * (modification time, change time, size, inode) differs from the one recorded when this process last reconciled
791
- * it; an unchanged file costs a stat, not a read and a hash, and `indexFile` skips an unchanged file by its hash.
792
- *
793
- * Which files a pass checks depends on who runs it. A save passes `touched`, the files it wrote (the day's note, the
794
- * ledger): then only those, the files still owed (unsettled, see `unsettledFiles`) and the ones the index has no
795
- * finished row for are stat'ed, so the vault is not walked and no other note's path is loaded, and a save costs the
796
- * same however many notes and events the workspace holds. Without `touched` the vault is walked and every file
797
- * stat'ed: that is what a search that finishes a save's unfinished files and a background job run, and what starts
798
- * each process. A save walks it too when the last full check is FULL_VAULT_CHECK_MS or more old, so an edit, a new
799
- * file, a rename or a delete made outside the engine is seen by a save within that long (and at once by
800
- * `index_memory`, a full pass that reads every file). A file changed within a clock tick of its scan is not recorded
801
- * as reconciled (see TIMESTAMP_TICK_MS), so the next check reads it again. Like a full pass it leaves a deleted or
802
- * renamed-away file's rows in the index. The one change a stamp cannot show is a rewrite that leaves all four fields
803
- * as they were, which takes setting the system clock back.
804
- */
805
- async function engineIndexChanged(db, config, engine,
806
- /** When the scan starts, on the clock the files' times are read against; a test passes a later one to age its files. */
807
- scanStart = Date.now(),
808
- /**
809
- * `interactive`: an agent waits on this pass (a save's, or a read that finishes what a save left), so a file whose
810
- * embedding the service will not give within the write budget is left unfinished and the pass returns; a background
811
- * job then embeds it (`engineFinishIndexInBackground`), however long the service takes.
812
- *
813
- * `touched`: the absolute paths of the files the calling save wrote. Given, the pass checks those and the files it
814
- * still owes, and walks no vault, unless the last full check is FULL_VAULT_CHECK_MS old or there has been none in this
815
- * process.
816
- */
817
- pass = {}) {
818
- const options = indexOptionsFor(engine, pass.interactive === true);
819
- let reconciled = reconciledStamps.get(db);
820
- if (!reconciled) {
821
- reconciled = new Map();
822
- reconciledStamps.set(db, reconciled);
823
- }
824
- const known = reconciled;
825
- let named = tooLargeStamps.get(db);
826
- if (!named) {
827
- named = new Map();
828
- tooLargeStamps.set(db, named);
829
- }
830
- const tooLarge = named;
831
- let unsettled = unsettledFiles.get(db);
832
- if (!unsettled) {
833
- unsettled = new Set();
834
- unsettledFiles.set(db, unsettled);
835
- }
836
- let candidates;
837
- let isFinished;
838
- const now = Date.now();
839
- const lastCheck = walkedAt.get(db);
840
- const checkedRecently = lastCheck !== undefined && now >= lastCheck && now - lastCheck < exports.FULL_VAULT_CHECK_MS;
841
- if (pass.touched !== undefined && checkedRecently) {
842
- // What the vault's rules leave out is never checked: the walk never reaches it, so neither does a save.
843
- const ignored = await (0, walker_1.vaultIgnores)(config.memoryDir);
844
- const owed = [...pass.touched, ...unsettled];
845
- if ((0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck().get() > 0) {
846
- owed.push(...(0, statements_1.prepareOnce)(db, "SELECT path FROM notes WHERE note_hash = ''").pluck().all());
847
- }
848
- candidates = (0, walker_1.statVaultFiles)([...new Set(owed)].filter(file => !ignored(file)));
849
- // A file that is gone or ignored has nothing left to settle: a later pass finds nothing there either.
850
- const present = new Set(candidates.map(f => f.file));
851
- for (const file of unsettled)
852
- if (!present.has(file))
853
- unsettled.delete(file);
854
- const hash = (0, statements_1.prepareOnce)(db, "SELECT note_hash FROM notes WHERE path = ?").pluck();
855
- isFinished = (file) => {
856
- const h = hash.get(file);
857
- return typeof h === 'string' && h !== '';
858
- };
859
- }
860
- else {
861
- candidates = await (0, walker_1.walkVaultStats)(config.memoryDir);
862
- const finished = new Set(db.prepare("SELECT path FROM notes WHERE note_hash != ''").pluck().all());
863
- isFinished = (file) => finished.has(file);
864
- walkedAt.set(db, now);
865
- unsettled.clear(); // rebuilt below from every file's stamp
866
- }
867
- const changed = candidates.filter(f => tooLarge.get(f.file) !== stampOf(f) && (!isFinished(f.file) || known.get(f.file) !== stampOf(f)));
868
- let indexed = 0;
869
- let errors = 0;
870
- let failure;
871
- // Together: each file that embeds waits on the provider for a round trip, and a save that changed the
872
- // daily note and the ledger should wait for one round trip, not two.
873
- await inPool(changed, exports.INDEX_FILE_CONCURRENCY, async (f) => {
874
- try {
875
- await (0, pipeline_1.indexFile)(db, f.file, options);
876
- if (stampIsTrusted(f, scanStart))
877
- known.set(f.file, stampOf(f));
878
- else
879
- known.delete(f.file);
880
- tooLarge.delete(f.file);
881
- indexed++;
882
- }
883
- catch (err) {
884
- if (err instanceof pipeline_1.NoteTooLargeError) {
885
- // Not a failure to retry: it stays out of the index until the file changes, and is said once per change.
886
- known.delete(f.file);
887
- tooLarge.set(f.file, stampOf(f));
888
- log.warn('note_too_large', { file: path_1.default.relative(config.memoryDir, f.file) }, err);
889
- return;
890
- }
891
- known.delete(f.file);
892
- if (err.code === 'ENOENT')
893
- return;
894
- errors++;
895
- failure ??= failureReason(err, 300);
896
- log.error('index_file_failed', { file: path_1.default.relative(config.memoryDir, f.file) }, err);
897
- }
898
- });
899
- // Settle what was checked: a file whose stamp is recorded (or that is named too large for its stamp) is done; any
900
- // other is owed, and the next save checks it again.
901
- for (const f of candidates) {
902
- if (known.get(f.file) === stampOf(f) || tooLarge.get(f.file) === stampOf(f))
903
- unsettled.delete(f.file);
904
- else
905
- unsettled.add(f.file);
906
- }
907
- // What the pass could not finish is not dropped: it is finished in the background, once the service answers.
908
- if (errors > 0 && pass.interactive)
909
- engineFinishIndexInBackground(db, config, engine);
910
- return { total: changed.length, indexed, errors, ...(failure === undefined ? {} : { failure }) };
911
- }
912
- /**
913
- * The files whose last index pass did not finish: their embedding failed, so keyword search finds their chunks and
914
- * search by meaning does not, until a later pass embeds them (see indexFile). A partial index answers it (migration
915
- * 15), so asking costs those few rows, usually none, however many notes there are.
916
- */
917
- exports.UNFINISHED_FILES = "SELECT COUNT(*) FROM notes WHERE note_hash = ''";
918
- /** After a search could not finish them, how long searches leave the unfinished files before trying again. */
919
- exports.UNFINISHED_RETRY_MS = 30_000;
920
- /** Per index: the epoch ms before which a search does not try the unfinished files again. */
921
- const unfinishedRetryAfter = new WeakMap();
922
- /**
923
- * Finish, for a search, the files a save left unfinished. A save whose embedding failed (the provider was down)
924
- * keeps its chunks, which keyword search finds, and leaves its file unfinished for a later pass to embed: the next
925
- * save's, or this one, which a search runs once the provider has embedded its query (`asked`: the search did, or, on
926
- * an index with no vector yet, which a search does not ask, it is asked here). So what was saved during an outage is
927
- * found by meaning as soon as the provider is back, not only after the next save; and a search waits on a provider
928
- * that is down no longer than for its own query, which then fails, so the search says semantic search failed rather
929
- * than that nothing matched. It is engineIndexChanged's full pass (no `touched`: every file is checked, as a save's first
930
- * pass in a process does). When it leaves a file unfinished although
931
- * the provider answers (a chunk it refuses), searches leave the files for UNFINISHED_RETRY_MS, so that costs a search a
932
- * provider call at most that often; saves go on retrying them as before. Throws only what embedding the query throws.
933
- * True when a file was finished, so the search runs again and finds its chunks.
934
- */
935
- async function finishUnfinishedFiles(db, memoryDir, engine,
936
- /** The provider as a read sees it: its calls end within the read budget. */
937
- provider, query, asked) {
938
- const unfinished = (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck();
939
- const before = unfinished.get();
940
- if (before === 0 || Date.now() < (unfinishedRetryAfter.get(db) ?? 0))
941
- return false;
942
- // Asked here only to learn that it answers: the search that follows embeds the query and reports it as before.
943
- if (!asked.queryVector)
944
- await (provider.embedQuery ? provider.embedQuery(query) : provider.embed([query]));
945
- const finish = async () => {
946
- try {
947
- await engineIndexChanged(db, { memoryDir }, engine, undefined, { interactive: true });
948
- }
949
- catch (err) {
950
- log.error('index_finish_failed', {}, err);
951
- }
952
- const after = unfinished.get();
953
- if (after > 0)
954
- unfinishedRetryAfter.set(db, Date.now() + exports.UNFINISHED_RETRY_MS);
955
- else
956
- unfinishedRetryAfter.delete(db);
957
- return after < before;
958
- };
959
- // The pass takes each file's lock, which a save that is embedding holds; a read does not wait for it beyond its own
960
- // budget. A pass that is still running then goes on without it, and the read is answered with what is indexed now.
961
- const budgetMs = engine.embedBudgets?.readMs;
962
- if (budgetMs === undefined || !Number.isFinite(budgetMs))
963
- return finish();
964
- const running = finish();
965
- let timer;
966
- const late = new Promise((resolve) => { timer = setTimeout(() => resolve(false), budgetMs); });
967
- try {
968
- const finished = await Promise.race([running, late]);
969
- if (!finished)
970
- unfinishedRetryAfter.set(db, Date.now() + exports.UNFINISHED_RETRY_MS);
971
- return finished;
972
- }
973
- finally {
974
- clearTimeout(timer);
975
- }
976
- }
977
- const backgroundIndexes = new WeakMap();
978
- /** The first wait is short: a save that just failed is usually a service that is already back. */
979
- exports.BACKGROUND_INDEX_FIRST_DELAY_MS = 1_000;
980
- /** After an attempt that failed: 30 s, doubling, up to 5 min (the wait a rebuild has). */
981
- const backgroundIndexRetryDelayMs = (outage) => Math.min(300_000, 30_000 * 2 ** outage);
982
- /**
983
- * Finish, in the background, what a save could not: the chunks it kept for keyword search that have no vector, and the
984
- * files left unfinished for them. The embedding is the provider's patient one (its own retries and backoff, however long
985
- * they take) and holds no lock, so a service that is down costs only this task's time; the pass that follows takes each
986
- * file's lock for as long as it takes to record that its chunks have their vectors. When an attempt fails it waits
987
- * (30 s, doubling, up to 5 min) and tries again, so a failed index is queued and never dropped, and it ends only when
988
- * nothing is left or `abort` is called. A job already running for this index is returned rather than started twice.
989
- * An index whose vectors are not whole (a rebuild is embedding all of them) is left to that rebuild.
990
- */
991
- function engineFinishIndexInBackground(db, config, engine, options = {}) {
992
- const { provider, vectorIndex } = engine;
993
- if (!provider || !vectorIndex)
994
- return null;
995
- const running = backgroundIndexes.get(db);
996
- if (running)
997
- return running;
998
- const { firstDelayMs = exports.BACKGROUND_INDEX_FIRST_DELAY_MS, retryDelayMs = backgroundIndexRetryDelayMs } = options;
999
- const controller = new AbortController();
1000
- const unfinished = (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck();
1001
- let waiting = false;
1002
- const wait = async (ms) => {
1003
- waiting = true;
1004
- try {
1005
- await (0, rebuild_1.sleep)(ms, controller.signal);
1006
- }
1007
- finally {
1008
- waiting = false;
1009
- }
1010
- };
1011
- const run = async () => {
1012
- await wait(firstDelayMs);
1013
- for (let outage = 0; !controller.signal.aborted; outage++) {
1014
- try {
1015
- if (semanticReady(engine)) {
1016
- await (0, rebuild_1.embedMissingVectors)(db, vectorIndex, provider, { signal: controller.signal });
1017
- if (controller.signal.aborted)
1018
- return;
1019
- // Their vectors are in: this records it, and embeds only what a save has changed since (within the write budget).
1020
- await engineIndexChanged(db, config, engine, undefined, { interactive: true });
1021
- if (unfinished.get() === 0 && (0, rebuild_1.chunksWithoutVectors)(db) === 0) {
1022
- if (outage > 0)
1023
- log.info('embeddings_recovered', { detail: 'the embedding service answers again; what was saved while it did not is findable by meaning now' });
1024
- return;
1025
- }
1026
- }
1027
- else if (!vectorIndex.rebuilding()) {
1028
- return; // another model's space: a rebuild is the only thing that can make it whole, and it is not this job's
1029
- }
1030
- throw new Error('the index is not finished yet');
1031
- }
1032
- catch (err) {
1033
- if (controller.signal.aborted)
1034
- return;
1035
- const delay = retryDelayMs(outage);
1036
- log.error('background_index_retry', { unembedded: (0, rebuild_1.chunksWithoutVectors)(db), retryInS: Math.round(delay / 1000) }, err);
1037
- await wait(delay);
1038
- }
1039
- }
1040
- };
1041
- const job = {
1042
- abort: () => controller.abort(),
1043
- waiting: () => waiting,
1044
- done: run().catch((err) => log.error('background_index_failed', {}, err)).finally(() => {
1045
- if (backgroundIndexes.get(db) === job)
1046
- backgroundIndexes.delete(db);
1047
- }),
1048
- };
1049
- backgroundIndexes.set(db, job);
1050
- return job;
1051
- }
1052
- /** The job `engineFinishIndexInBackground` started for this index that has not ended yet, or null; starts none. */
1053
- function runningBackgroundIndex(db) {
1054
- return backgroundIndexes.get(db) ?? null;
1055
- }
1056
- /**
1057
- * Whether anything is waiting for the embedding service: a file left unfinished, or a chunk with no vector in an index
1058
- * that is otherwise whole. Asked at start, so what an earlier process (or an idle exit) left is finished.
1059
- */
1060
- function hasIndexWorkToFinish(db, engine) {
1061
- if (!engine.provider || !engine.vectorIndex || !semanticReady(engine))
1062
- return false;
1063
- return (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck().get() > 0 || (0, rebuild_1.chunksWithoutVectors)(db) > 0;
1064
- }
1065
- async function engineRememberNote(config, text, date, source, scope,
1066
- /** The user's time zone, when the client names it: the note is then filed under their day, at their time. */
1067
- timeZone) {
1068
- // One reading of the clock gives the entry both its day and its time.
1069
- const clock = (0, writer_1.entryClock)(new Date(), timeZone);
1070
- const day = date ?? clock.date;
1071
- const dir = (0, scope_1.dailyDirForScope)((0, scope_1.normalizeScope)(scope));
1072
- const written = await (0, writer_1.appendDailyEntry)(config.memoryDir, day, { text, source: source ?? 'mcp' }, dir, clock);
1073
- const file = clientPath(config.memoryDir, written.filePath);
1074
- // The id that reads, retires or restores this entry (memory_map's), so forgetting it needs no lookup (live dogfood
1075
- // 2026-10-03, D12).
1076
- const id = (0, memoryMap_1.appendedEntryId)(file.split(path_1.default.sep).join('/'), written.entry);
1077
- return { ok: true, file, scope: (0, scope_1.normalizeScope)(scope), ...(id ? { id } : {}) };
1078
- }