renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,350 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.DigestWorker = exports.DigestBudgetError = exports.DIGEST_MAX_FAILED_SWEEPS = exports.DIGEST_RETRY_MAX_MS = exports.DIGEST_RETRY_BASE_MS = exports.DIGEST_MIN_INTERVAL_MS = exports.DIGEST_DEBOUNCE_MS = void 0;
7
- exports.guardedDigestChat = guardedDigestChat;
8
- const node_fs_1 = __importDefault(require("node:fs"));
9
- const sessionRouter_1 = require("../router/sessionRouter");
10
- const digestCache_1 = require("./digestCache");
11
- const digests_1 = require("./digests");
12
- const log_1 = require("../mvp/log");
13
- const log = (0, log_1.createLogger)('engine');
14
- /**
15
- * The production path of write-time digests (RENOR_DIGESTS, see digests.ts): a background worker that writes the
16
- * digest of every note that has none, after a save, without ever being on the save's path.
17
- *
18
- * - `notify()` (called after a save's index pass) only sets a timer and returns: a save never waits for a model.
19
- * - Saves are batched: the sweep runs DIGEST_DEBOUNCE_MS after the last save, and no sooner than
20
- * DIGEST_MIN_INTERVAL_MS after the previous sweep ended, so a burst of saves, or one note appended to all day, is
21
- * digested once per interval and not once per save. One sweep runs at a time.
22
- * - A sweep that left notes undigested because the provider failed is tried again after a wait that doubles from
23
- * DIGEST_RETRY_BASE_MS to DIGEST_RETRY_MAX_MS (with jitter), up to DIGEST_MAX_FAILED_SWEEPS in a row; then the
24
- * worker stops until the next start (a save tries once more) rather than calling a provider that keeps refusing.
25
- * The transport inside a call retries on its own too (createOllamaChat, RENOR_DIGEST_RETRIES).
26
- * - Cost guard, per workspace: every call is counted at the list price in the workspace's own digest cache
27
- * (`digest_spend`), a call that could take the UTC day past RENOR_DIGEST_MAX_USD_PER_DAY is not made (its worst case
28
- * is counted: the prompt plus the whole output allowance), and the worker is `paused-budget` until the day after the
29
- * one that refused the call (a sweep that ends past midnight does not move that). Notes without a digest wait; nothing
30
- * is lost and search reads the digests that exist. Only this worker is held to the guard: `renor index` and
31
- * `renor rebuild` are commands someone runs and are not capped (README, "Cost guard").
32
- * - A digest of text a note no longer holds (edited, forgotten, file gone) is removed at the start of every sweep. A note
33
- * is edited once the index has seen it (a save, an import, `renor index`): a file changed behind the index's back keeps
34
- * its digest, as search keeps its old chunks, until then.
35
- * - The only provider is the digest slot (RENOR_DIGEST_BASE_URL + RENOR_DIGEST_API_KEY). With it unset the worker is
36
- * `stopped` and nothing is sent: it never falls back to the evidence provider, which the privacy copy does not name.
37
- * - The sweep is held in the on-demand idle monitor while it runs, so a workspace does not exit in the middle of a call.
38
- * Waits for a retry are not held: the workspace may sleep, and its next start sweeps again.
39
- *
40
- * Nothing here runs unless RENOR_DIGESTS is on, and it is off by default.
41
- */
42
- exports.DIGEST_DEBOUNCE_MS = 15_000;
43
- exports.DIGEST_MIN_INTERVAL_MS = 60_000;
44
- exports.DIGEST_RETRY_BASE_MS = 30_000;
45
- exports.DIGEST_RETRY_MAX_MS = 300_000;
46
- exports.DIGEST_MAX_FAILED_SWEEPS = 8;
47
- /** How many days of spend rows are kept. */
48
- const SPEND_DAYS_KEPT = 14;
49
- /** Raised, inside the model client, when a call would take the day past its budget. */
50
- class DigestBudgetError extends Error {
51
- constructor(message) {
52
- super(message);
53
- this.name = 'DigestBudgetError';
54
- }
55
- }
56
- exports.DigestBudgetError = DigestBudgetError;
57
- const utcDay = (ms) => new Date(ms).toISOString().slice(0, 10);
58
- /** Roughly three characters a token for the prompt: the worst case a guard must hold, not a measurement. */
59
- function promptTokensOf(request) {
60
- return Math.ceil(request.messages.reduce((sum, m) => sum + m.content.length, 0) / 3);
61
- }
62
- /**
63
- * The cost guard around the digest model client: refuses a call whose worst case would take the day past the budget, and
64
- * records what each call cost. A call that threw is counted at its prompt alone (what is known to have been sent).
65
- */
66
- function guardedDigestChat(args) {
67
- const { chat, cache, options } = args;
68
- const usd = (promptTokens, outputTokens) => (promptTokens * options.priceInPerM + outputTokens * options.priceOutPerM) / 1e6;
69
- return async (request) => {
70
- const day = args.day();
71
- const promptEstimate = promptTokensOf(request);
72
- const worst = usd(promptEstimate, request.max_tokens ?? 8_000);
73
- if (cache.spentUsd(day) + worst > options.maxUsdPerDay) {
74
- args.onBudget(day);
75
- throw new DigestBudgetError(`the digest budget for ${day} (${options.maxUsdPerDay} USD) would be passed`);
76
- }
77
- let response;
78
- try {
79
- response = await chat(request);
80
- }
81
- catch (err) {
82
- cache.addSpend(day, promptEstimate, 0, usd(promptEstimate, 0));
83
- throw err;
84
- }
85
- const prompt = response.promptTokens ?? promptEstimate;
86
- const output = (0, sessionRouter_1.billedOutputTokens)(response);
87
- cache.addSpend(day, prompt, output, usd(prompt, output));
88
- return response;
89
- };
90
- }
91
- class DigestWorker {
92
- deps;
93
- now;
94
- setTimer;
95
- clearTimer;
96
- random;
97
- log;
98
- state = 'idle';
99
- reason;
100
- timer = null;
101
- timerAt = 0;
102
- running = null;
103
- dirty = false;
104
- closed = false;
105
- failures = 0;
106
- lastEnd = 0;
107
- budgetHit = false;
108
- /** The UTC day whose budget refused a call in the sweep in flight: the pause is counted from it, not from when the sweep ends. */
109
- budgetDay;
110
- last;
111
- constructor(deps) {
112
- this.deps = deps;
113
- this.now = deps.now ?? Date.now;
114
- this.setTimer = deps.setTimer ?? ((fn, ms) => {
115
- const handle = setTimeout(fn, ms);
116
- handle.unref();
117
- return handle;
118
- });
119
- this.clearTimer = deps.clearTimer ?? (handle => clearTimeout(handle));
120
- this.random = deps.random ?? Math.random;
121
- this.log = deps.log ?? (line => log.info('digests_worker', { detail: line }));
122
- }
123
- /** Boot: digest what earlier saves, an import, or a workspace that slept left without a digest. */
124
- start() {
125
- this.failures = 0;
126
- if (!this.runnable())
127
- return;
128
- this.schedule(exports.DIGEST_DEBOUNCE_MS);
129
- }
130
- /** After a save has been indexed. Never throws, never waits: it sets a timer. */
131
- notify() {
132
- if (this.closed || !this.runnable())
133
- return;
134
- if (this.running) {
135
- this.dirty = true;
136
- return;
137
- }
138
- // A retry already waiting keeps its (longer) wait: a save does not shorten a backoff. A stopped worker tries once.
139
- if (this.state === 'retrying' || this.state === 'paused-budget')
140
- return;
141
- this.schedule(exports.DIGEST_DEBOUNCE_MS);
142
- }
143
- status() {
144
- const cache = this.openCache();
145
- let spent = 0;
146
- try {
147
- spent = cache?.spentUsd(utcDay(this.now())) ?? 0;
148
- }
149
- finally {
150
- cache?.close();
151
- }
152
- return {
153
- state: this.state,
154
- ...(this.reason ? { reason: this.reason } : {}),
155
- consecutiveFailures: this.failures,
156
- spentTodayUsd: Math.round(spent * 1e6) / 1e6,
157
- maxUsdPerDay: this.deps.options.maxUsdPerDay,
158
- ...(this.last ? { lastSweep: this.last } : {}),
159
- };
160
- }
161
- /** Resolves when the sweep in flight, if any, has ended (shutdown, tests). Waits for no timer. */
162
- async whenSwept() {
163
- while (this.running)
164
- await this.running;
165
- }
166
- /** Run a sweep now (no debounce, no interval) and wait for it: boot-time catch-up, tests. */
167
- async sweepNow() {
168
- if (this.closed || !this.runnable())
169
- return;
170
- this.cancelTimer();
171
- if (this.running)
172
- await this.running;
173
- this.launch();
174
- await this.running;
175
- }
176
- /** Stop timers and wait for the sweep in flight; after this the worker does nothing. */
177
- async close() {
178
- this.closed = true;
179
- this.cancelTimer();
180
- if (this.running)
181
- await this.running;
182
- }
183
- runnable() {
184
- if (!this.deps.provider) {
185
- this.stop('no embedding provider: digests are read by meaning, so they are not written');
186
- return false;
187
- }
188
- if (!this.deps.options.chat) {
189
- // The digest slot is the only provider digests use: another provider's key is never borrowed (see resolveDigestProvider).
190
- this.stop('no digest model is configured: digests need RENOR_DIGEST_BASE_URL and RENOR_DIGEST_API_KEY, and never use the evidence or Ollama provider instead, so nothing is sent');
191
- return false;
192
- }
193
- return true;
194
- }
195
- stop(reason) {
196
- if (this.state === 'stopped' && this.reason === reason)
197
- return;
198
- this.state = 'stopped';
199
- this.reason = reason;
200
- this.log(`[digests] worker stopped: ${reason}`);
201
- }
202
- cancelTimer() {
203
- if (this.timer !== null)
204
- this.clearTimer(this.timer);
205
- this.timer = null;
206
- }
207
- schedule(wait, state = 'scheduled') {
208
- if (this.closed)
209
- return;
210
- const earliest = this.lastEnd + exports.DIGEST_MIN_INTERVAL_MS - this.now();
211
- const delay = Math.max(wait, state === 'scheduled' ? earliest : 0);
212
- const at = this.now() + delay;
213
- // One timer: a nearer request replaces a later one, a later request leaves the nearer one alone.
214
- if (this.timer !== null && this.timerAt <= at && this.state === state)
215
- return;
216
- this.cancelTimer();
217
- this.state = state;
218
- if (state === 'scheduled')
219
- this.reason = undefined;
220
- this.timerAt = at;
221
- this.timer = this.setTimer(() => {
222
- this.timer = null;
223
- this.launch();
224
- }, delay);
225
- }
226
- launch() {
227
- if (this.closed || this.running)
228
- return;
229
- const track = this.deps.track ?? (work => work());
230
- this.state = 'running';
231
- const run = this.sweep().catch((err) => {
232
- // A sweep that threw (the database closed under it, a disk error) is a failed sweep, said once and retried like any.
233
- this.failures++;
234
- this.reason = `sweep failed: ${String(err instanceof Error ? err.message : err).replace(/\s+/g, ' ').slice(0, 160)}`;
235
- this.log(`[digests] ${this.reason}`);
236
- }).then(() => this.afterSweep());
237
- this.running = run.finally(() => { this.running = null; });
238
- void track(() => this.running).catch(() => undefined);
239
- }
240
- afterSweep() {
241
- this.lastEnd = this.now();
242
- if (this.closed)
243
- return;
244
- if (this.budgetHit) {
245
- this.budgetHit = false;
246
- // The day that is spent is the one that refused the call, whenever the sweep ended: one that runs past UTC midnight
247
- // after the refusal must not push the resume out a whole day further. A resume time already past waits one second.
248
- const refusedOn = this.budgetDay ?? utcDay(this.now());
249
- this.budgetDay = undefined;
250
- const resume = Date.parse(`${refusedOn}T00:00:00Z`) + 86_400_000 + 60_000;
251
- this.state = 'paused-budget';
252
- this.reason = `the daily digest budget (${this.deps.options.maxUsdPerDay} USD) is spent; notes without a digest wait until the next UTC day`;
253
- this.log(`[digests] ${this.reason}`);
254
- this.dirty = false;
255
- this.cancelTimer();
256
- this.timerAt = resume;
257
- this.timer = this.setTimer(() => {
258
- this.timer = null;
259
- this.state = 'idle';
260
- this.schedule(0);
261
- }, Math.max(1_000, resume - this.now()));
262
- return;
263
- }
264
- if (this.failures > 0) {
265
- if (this.failures >= exports.DIGEST_MAX_FAILED_SWEEPS) {
266
- this.dirty = false;
267
- this.stop(`the digest model failed ${this.failures} sweeps in a row (${this.reason ?? 'no detail'}); it is tried again at the next start or save`);
268
- return;
269
- }
270
- const wait = Math.min(exports.DIGEST_RETRY_MAX_MS, exports.DIGEST_RETRY_BASE_MS * 2 ** (this.failures - 1)) * (0.8 + 0.4 * this.random());
271
- this.dirty = false;
272
- this.log(`[digests] ${this.reason ?? 'a sweep failed'}; trying again in ${Math.round(wait / 1000)} s`);
273
- this.schedule(wait, 'retrying');
274
- return;
275
- }
276
- this.state = 'idle';
277
- this.reason = undefined;
278
- if (this.dirty) {
279
- this.dirty = false;
280
- this.schedule(exports.DIGEST_DEBOUNCE_MS);
281
- }
282
- }
283
- openCache() {
284
- try {
285
- return new digestCache_1.DigestCache(this.deps.options.cachePath);
286
- }
287
- catch (err) {
288
- this.log(`[digests] the digest cache cannot be opened: ${String(err instanceof Error ? err.message : err).slice(0, 160)}`);
289
- return null;
290
- }
291
- }
292
- async sweep() {
293
- const { db, config, options, provider } = this.deps;
294
- const cache = this.openCache();
295
- if (!cache)
296
- throw new Error('the digest cache cannot be opened');
297
- try {
298
- const ledger = config.ledgerRelPath;
299
- // What search can still reach: notes whose file exists, by the hash the INDEX holds. A digest of any other text is
300
- // removed. A file edited behind the index's back keeps its index hash, so it keeps its digest, as search keeps its
301
- // old chunks, until the index sees the change (a save, an import, `renor index`).
302
- const live = (0, sessionRouter_1.listRouterNotes)(db, config.memoryDir, ledger).filter(note => node_fs_1.default.existsSync(note.file));
303
- const pruned = cache.retainOnly(new Set(live.map(note => note.noteHash)));
304
- cache.pruneSpend(SPEND_DAYS_KEPT);
305
- this.budgetHit = false;
306
- this.budgetDay = undefined;
307
- const guarded = {
308
- ...options,
309
- chat: guardedDigestChat({
310
- chat: options.chat,
311
- cache,
312
- day: () => utcDay(this.now()),
313
- options,
314
- onBudget: (day) => { this.budgetHit = true; this.budgetDay = day; },
315
- }),
316
- };
317
- const stats = await (0, digests_1.buildNoteDigests)({
318
- db,
319
- vaultPath: config.memoryDir,
320
- ledgerRelPath: ledger,
321
- options: guarded,
322
- cache,
323
- provider: provider,
324
- });
325
- if (stats.failed > 0 && !this.budgetHit) {
326
- this.failures++;
327
- this.reason = (stats.error ?? 'the digest model left notes without a digest').replace(/\s+/g, ' ').slice(0, 160);
328
- }
329
- else if (stats.failed === 0) {
330
- this.failures = 0;
331
- this.reason = undefined;
332
- }
333
- this.last = {
334
- endedAt: new Date(this.now()).toISOString(),
335
- digested: stats.digested,
336
- failed: stats.failed,
337
- gone: stats.gone,
338
- calls: stats.calls,
339
- pruned,
340
- };
341
- if (stats.digested > 0 || stats.failed > 0 || pruned > 0) {
342
- this.log(`[digests] swept: ${stats.digested} digested, ${stats.failed} failed, ${stats.cached} already had one, ${pruned} removed, ${stats.calls} call(s)`);
343
- }
344
- }
345
- finally {
346
- cache.close();
347
- }
348
- }
349
- }
350
- exports.DigestWorker = DigestWorker;
@@ -1,337 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.CacheOnlyVertex = exports.CountingEmbeddingProvider = exports.CachingEmbeddingProvider = void 0;
7
- exports.embeddingCallStats = embeddingCallStats;
8
- exports.resolveEmbeddingCachePath = resolveEmbeddingCachePath;
9
- exports.describeEmbeddingCache = describeEmbeddingCache;
10
- exports.invalidateEmbeddingCache = invalidateEmbeddingCache;
11
- const node_fs_1 = __importDefault(require("node:fs"));
12
- const node_path_1 = __importDefault(require("node:path"));
13
- const node_crypto_1 = __importDefault(require("node:crypto"));
14
- const sqlite_1 = __importDefault(require("../db/sqlite"));
15
- const IDENTITY_KIND = /^[\w.-]{1,32}$/;
16
- const IDENTITY_MODEL = /^[\w./:-]{1,80}$/;
17
- function resolveIdentity(inner, stated) {
18
- const declared = inner.identity;
19
- const identity = stated ?? declared;
20
- if (!identity)
21
- throw new Error('Embedding cache needs the identity of the provider and model: declare `identity` on the provider or pass options.identity');
22
- if (!IDENTITY_KIND.test(identity.kind) || !IDENTITY_MODEL.test(identity.model))
23
- throw new Error('Invalid embedding identity');
24
- if (stated && declared && (stated.kind !== declared.kind || stated.model !== declared.model)) {
25
- throw new Error(`Embedding cache identity ${stated.kind}/${stated.model} contradicts the wrapped provider, which declares ${declared.kind}/${declared.model}`);
26
- }
27
- return identity;
28
- }
29
- const TASK_QUERY = 'RETRIEVAL_QUERY';
30
- const TASK_DOCUMENT = 'RETRIEVAL_DOCUMENT';
31
- class CachingEmbeddingProvider {
32
- dimensions;
33
- identity;
34
- /** Lookups: one hit or miss per input occurrence. What the provider received is `sent`. */
35
- stats = { hits: 0, misses: 0 };
36
- sent = { texts: 0, calls: 0 };
37
- inner;
38
- db;
39
- namespace;
40
- insert;
41
- lookup;
42
- oldest;
43
- remove;
44
- maxEntries;
45
- maxBytes;
46
- serving;
47
- /** Rows and vector bytes held, kept in step with the file so a write does not scan it. */
48
- entries = 0;
49
- bytes = 0;
50
- /** Serving mode only: set by the first cache fault, after which every call goes straight to the provider. */
51
- disabled = false;
52
- constructor(inner, options) {
53
- if ('namespace' in options) {
54
- throw new Error('CachingEmbeddingProvider no longer takes a namespace: the identity of the provider and model pins the cache (declare `identity` on the provider, or pass options.identity)');
55
- }
56
- this.inner = inner;
57
- this.dimensions = inner.dimensions;
58
- this.identity = resolveIdentity(inner, options.identity);
59
- if (options.scope !== undefined && !/^[\w.-]{1,32}$/.test(options.scope))
60
- throw new Error('Invalid cache scope');
61
- this.namespace = `${this.identity.kind}/${this.identity.model}/${inner.dimensions}${options.scope ? `/${options.scope}` : ''}`;
62
- if (!/^[\w./:-]{4,120}$/.test(this.namespace))
63
- throw new Error('Invalid cache namespace');
64
- for (const [name, limit] of [['maxEntries', options.maxEntries], ['maxBytes', options.maxBytes]]) {
65
- if (limit !== undefined && (!Number.isInteger(limit) || limit < 1))
66
- throw new Error(`${name} must be a positive integer`);
67
- }
68
- this.maxEntries = options.maxEntries ?? Infinity;
69
- this.maxBytes = options.maxBytes ?? Infinity;
70
- this.serving = options.serving === true;
71
- node_fs_1.default.mkdirSync(node_path_1.default.dirname(options.cachePath), { recursive: true });
72
- this.db = new sqlite_1.default(options.cachePath);
73
- this.db.pragma('journal_mode = WAL');
74
- // A serving read waits a moment for a lock, not ten seconds: past that the provider is the faster answer.
75
- this.db.pragma(`busy_timeout = ${this.serving ? 100 : 10000}`);
76
- this.db.exec(`CREATE TABLE IF NOT EXISTS embeddings (
77
- key TEXT PRIMARY KEY,
78
- dims INTEGER NOT NULL,
79
- vec BLOB NOT NULL,
80
- namespace TEXT NOT NULL,
81
- created_at TEXT NOT NULL
82
- )`);
83
- this.lookup = this.db.prepare('SELECT vec, dims FROM embeddings WHERE key = ? AND namespace = ?');
84
- this.insert = this.db.prepare('INSERT OR IGNORE INTO embeddings (key, dims, vec, namespace, created_at) VALUES (?, ?, ?, ?, ?)');
85
- // rowid is insertion order (nothing but the oldest row is ever deleted), so eviction is deterministic.
86
- this.oldest = this.db.prepare('SELECT rowid AS id, LENGTH(vec) AS n FROM embeddings ORDER BY rowid LIMIT 32');
87
- this.remove = this.db.prepare('DELETE FROM embeddings WHERE rowid = ?');
88
- if (this.bounded) {
89
- const held = this.db.prepare('SELECT COUNT(*) AS entries, COALESCE(SUM(LENGTH(vec)), 0) AS bytes FROM embeddings').get();
90
- this.entries = held.entries;
91
- this.bytes = held.bytes;
92
- this.evict();
93
- }
94
- }
95
- get bounded() {
96
- return this.maxEntries !== Infinity || this.maxBytes !== Infinity;
97
- }
98
- evict() {
99
- while (this.entries > this.maxEntries || this.bytes > this.maxBytes) {
100
- const rows = this.oldest.all();
101
- if (rows.length === 0) {
102
- this.entries = 0;
103
- this.bytes = 0;
104
- return;
105
- }
106
- for (const row of rows) {
107
- if (this.entries <= this.maxEntries && this.bytes <= this.maxBytes)
108
- return;
109
- this.remove.run(row.id);
110
- this.entries -= 1;
111
- this.bytes -= row.n;
112
- }
113
- }
114
- }
115
- /** Serving mode: a cache fault ends the cache for this process and the call carries on without it. */
116
- guard(work, fallback) {
117
- if (!this.serving)
118
- return work();
119
- if (this.disabled)
120
- return fallback;
121
- try {
122
- return work();
123
- }
124
- catch {
125
- this.disabled = true;
126
- try {
127
- this.db.close();
128
- }
129
- catch { /* already unusable */ }
130
- return fallback;
131
- }
132
- }
133
- key(text, taskType) {
134
- // A serving entry holds float64, a benchmark entry float32: they never share a key.
135
- return node_crypto_1.default.createHash('sha256')
136
- .update(`${this.namespace}\0${taskType}\0${this.dimensions}\0${text}${this.serving ? '\0f64' : ''}`)
137
- .digest('hex');
138
- }
139
- read(text, taskType) {
140
- const row = this.lookup.get(this.key(text, taskType), this.namespace);
141
- if (!row)
142
- return null;
143
- if (row.dims !== this.dimensions)
144
- return null; // impossible unless the store was corrupted
145
- const width = this.serving ? 8 : 4;
146
- if (row.vec.byteLength !== this.dimensions * width)
147
- return null;
148
- // Copied out first: a Buffer from the driver is not guaranteed to start on an aligned boundary.
149
- const bytes = row.vec.buffer.slice(row.vec.byteOffset, row.vec.byteOffset + row.vec.byteLength);
150
- return Array.from(this.serving ? new Float64Array(bytes) : new Float32Array(bytes));
151
- }
152
- write(text, taskType, vector) {
153
- if (vector.length !== this.dimensions)
154
- throw new Error('Cached vector has wrong dimensionality');
155
- const buf = Buffer.from((this.serving ? new Float64Array(vector) : new Float32Array(vector)).buffer);
156
- const { changes } = this.insert.run(this.key(text, taskType), this.dimensions, buf, this.namespace, new Date().toISOString());
157
- if (changes > 0 && this.bounded) {
158
- this.entries += 1;
159
- this.bytes += buf.length;
160
- this.evict();
161
- }
162
- }
163
- async embed(texts, options) {
164
- if (texts.length === 0)
165
- return [];
166
- if (this.serving)
167
- return this.inner.embed(texts, options); // documents are not cached on the read path
168
- const out = [];
169
- const misses = new Map();
170
- texts.forEach((text, index) => {
171
- const cached = this.read(text, TASK_DOCUMENT);
172
- if (cached) {
173
- this.stats.hits += 1;
174
- out[index] = cached;
175
- }
176
- else {
177
- this.stats.misses += 1;
178
- const indexes = misses.get(text);
179
- if (indexes)
180
- indexes.push(index);
181
- else
182
- misses.set(text, [index]);
183
- }
184
- });
185
- if (misses.size > 0) {
186
- const uniqueTexts = [...misses.keys()];
187
- this.sent.calls += 1;
188
- this.sent.texts += uniqueTexts.length;
189
- const fresh = await this.inner.embed(uniqueTexts, options);
190
- if (fresh.length !== uniqueTexts.length)
191
- throw new Error('Inner provider returned wrong vector count');
192
- uniqueTexts.forEach((text, i) => {
193
- this.write(text, TASK_DOCUMENT, fresh[i]);
194
- for (const index of misses.get(text))
195
- out[index] = fresh[i];
196
- });
197
- }
198
- return out;
199
- }
200
- async embedQuery(text, options) {
201
- const cached = this.guard(() => this.read(text, TASK_QUERY), null);
202
- if (cached) {
203
- this.stats.hits += 1;
204
- return cached;
205
- }
206
- this.stats.misses += 1;
207
- this.sent.calls += 1;
208
- this.sent.texts += 1;
209
- // Providers without a query task type (OpenAI-compatible, hash) are queried
210
- // through embed(), exactly as retrieval/semantic.ts does when uncached.
211
- const vector = this.inner.embedQuery
212
- ? await this.inner.embedQuery(text, options)
213
- : (await this.inner.embed([text], options))[0];
214
- if (!this.serving)
215
- this.write(text, TASK_QUERY, vector);
216
- else
217
- this.guard(() => this.write(text, TASK_QUERY, vector), undefined);
218
- return vector;
219
- }
220
- get callStats() {
221
- return { hits: this.stats.hits, misses: this.stats.misses, cached: true, sent: { ...this.sent } };
222
- }
223
- close() {
224
- if (this.db.open)
225
- this.db.close();
226
- }
227
- describe() {
228
- const inner = this.inner.describe?.() ?? {};
229
- return { ...inner, cache: { namespace: this.namespace, hits: this.stats.hits, misses: this.stats.misses } };
230
- }
231
- }
232
- exports.CachingEmbeddingProvider = CachingEmbeddingProvider;
233
- /**
234
- * Stands in for the cache when it is off, so the texts a run sends to its provider are
235
- * still counted: an uncached run must not be reported as free.
236
- */
237
- class CountingEmbeddingProvider {
238
- dimensions;
239
- identity;
240
- sent = { texts: 0, calls: 0 };
241
- inner;
242
- constructor(inner) {
243
- this.inner = inner;
244
- this.dimensions = inner.dimensions;
245
- this.identity = inner.identity;
246
- }
247
- async embed(texts, options) {
248
- // An empty batch is no provider call, exactly as in CachingEmbeddingProvider.embed: the counter and the real
249
- // call boundary agree (every shipped provider already returns [] for it without a request).
250
- if (texts.length === 0)
251
- return [];
252
- this.sent.calls += 1;
253
- this.sent.texts += texts.length;
254
- return this.inner.embed(texts, options);
255
- }
256
- async embedQuery(text, options) {
257
- this.sent.calls += 1;
258
- this.sent.texts += 1;
259
- // Providers without a query task type are queried through embed(), exactly as retrieval/semantic.ts does.
260
- return this.inner.embedQuery ? this.inner.embedQuery(text, options) : (await this.inner.embed([text], options))[0];
261
- }
262
- /** Nothing is looked up, so every text sent is a miss and none is a hit. */
263
- get callStats() {
264
- return { hits: 0, misses: this.sent.texts, cached: false, sent: { ...this.sent } };
265
- }
266
- }
267
- exports.CountingEmbeddingProvider = CountingEmbeddingProvider;
268
- /** A run's embedding figures for the cache report: cache counters, or the count of texts sent when no cache stood in front. */
269
- function embeddingCallStats(provider) {
270
- return provider instanceof CachingEmbeddingProvider || provider instanceof CountingEmbeddingProvider ? provider.callStats : undefined;
271
- }
272
- /** Resolve the cache setting from the environment. Returns null when disabled. */
273
- function resolveEmbeddingCachePath(env = process.env) {
274
- const raw = env.RENOR_EVAL_EMBED_CACHE;
275
- if (raw === 'off' || raw === '0')
276
- return null;
277
- if (!raw || raw === 'on') {
278
- return node_path_1.default.join(process.env.HOME ?? '.', '.local', 'share', 'renor', 'embed-cache.db');
279
- }
280
- return node_path_1.default.resolve(raw);
281
- }
282
- /** Cached row count per namespace (a missing DB is an empty cache). */
283
- function describeEmbeddingCache(cachePath) {
284
- if (!node_fs_1.default.existsSync(cachePath))
285
- return [];
286
- const db = new sqlite_1.default(cachePath, { readonly: true });
287
- try {
288
- return db.prepare('SELECT namespace, COUNT(*) AS rows FROM embeddings GROUP BY namespace ORDER BY namespace').all();
289
- }
290
- finally {
291
- db.close();
292
- }
293
- }
294
- /**
295
- * Invalidate cached vectors by namespace (kind/model/dims), or everything with
296
- * an explicit `all`. An empty filter is refused so a typo never wipes a paid cache.
297
- */
298
- function invalidateEmbeddingCache(cachePath, filter) {
299
- if (!filter.namespace && !filter.all)
300
- throw new Error('invalidateEmbeddingCache: pass a namespace or all: true');
301
- if (!node_fs_1.default.existsSync(cachePath))
302
- return 0;
303
- const db = new sqlite_1.default(cachePath);
304
- try {
305
- db.pragma('busy_timeout = 10000');
306
- return filter.namespace
307
- ? db.prepare('DELETE FROM embeddings WHERE namespace = ?').run(filter.namespace).changes
308
- : db.prepare('DELETE FROM embeddings').run().changes;
309
- }
310
- finally {
311
- db.close();
312
- }
313
- }
314
- /** Read existing benchmark Vertex vectors without creating a cache or supplying an upstream. */
315
- class CacheOnlyVertex {
316
- dimensions = 768;
317
- identity = { kind: 'vertex', model: 'gemini-embedding-001' };
318
- stats = { hits: 0, misses: 0, upstreamTexts: 0 };
319
- db;
320
- constructor(file) { this.db = new sqlite_1.default(file, { readonly: true, fileMustExist: true }); }
321
- read(text, task) {
322
- const namespace = 'vertex/gemini-embedding-001/768';
323
- const key = node_crypto_1.default.createHash('sha256').update(`${namespace}\0${task}\0${this.dimensions}\0${text}`).digest('hex');
324
- const row = this.db.prepare('SELECT vec, dims FROM embeddings WHERE key = ? AND namespace = ?').get(key, namespace);
325
- if (!row || row.dims !== 768 || row.vec.length !== 768 * 4) {
326
- this.stats.misses++;
327
- throw new Error(`cache-only Vertex miss: task=${task} textSHA256=${node_crypto_1.default.createHash('sha256').update(text).digest('hex')}`);
328
- }
329
- this.stats.hits++;
330
- const bytes = row.vec.buffer.slice(row.vec.byteOffset, row.vec.byteOffset + row.vec.byteLength);
331
- return Array.from(new Float32Array(bytes));
332
- }
333
- async embed(texts) { return texts.map(text => this.read(text, 'RETRIEVAL_DOCUMENT')); }
334
- async embedQuery(text) { return this.read(text, 'RETRIEVAL_QUERY'); }
335
- close() { this.db.close(); }
336
- }
337
- exports.CacheOnlyVertex = CacheOnlyVertex;