renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,177 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.CoalescingEmbeddingProvider = exports.COALESCE_REQUESTS_IN_FLIGHT = exports.COALESCE_REQUEST_TEXTS = void 0;
4
- const types_1 = require("./types");
5
- /** Texts per request to the service: one predict call takes this many (the Vertex provider splits at the same size). */
6
- exports.COALESCE_REQUEST_TEXTS = 32;
7
- /** Requests in flight at once: the same as the rebuild's concurrency, so the service sees no more of them than before. */
8
- exports.COALESCE_REQUESTS_IN_FLIGHT = 4;
9
- /** How long a request waits for more callers when the queue is short and a request slot is free. */
10
- const GATHER_MS = 2;
11
- /**
12
- * Batches the document embeddings of concurrent callers into shared requests.
13
- *
14
- * An index pass embeds one note at a time, a request of three or four texts, so a pass over N notes waited on about N
15
- * round trips. This provider takes the texts of every caller that is waiting, in arrival order, and sends them as one
16
- * request of up to COALESCE_REQUEST_TEXTS texts, with at most COALESCE_REQUESTS_IN_FLIGHT requests out at once. Each caller
17
- * gets back exactly the vectors of its own texts, in its own order, so what it stores is what it would have stored alone.
18
- *
19
- * Failures stay per caller. A caller that stops waiting (its budget ran out, or its signal aborted) is dropped from the queue,
20
- * or from the request if it is already out, and the request is aborted once no caller of it still waits. When a request
21
- * fails for a reason other than the service being unavailable (a text it refuses, say), each of its callers is sent again
22
- * alone, as it would have been without batching, so one refused text fails one note and no other.
23
- *
24
- * Queries do not wait for a batch: `embedQuery` goes straight to the provider.
25
- */
26
- class CoalescingEmbeddingProvider {
27
- dimensions;
28
- identity;
29
- inner;
30
- queue = [];
31
- inFlight = 0;
32
- gatherTimer;
33
- gatherDone = false;
34
- constructor(inner) {
35
- this.inner = inner;
36
- this.dimensions = inner.dimensions;
37
- this.identity = inner.identity;
38
- }
39
- embed(texts, options) {
40
- if (texts.length === 0)
41
- return Promise.resolve([]);
42
- // An abort that came first is never heard as an event, so it is checked here.
43
- if (options?.signal?.aborted)
44
- return Promise.reject(options.signal.reason);
45
- return new Promise((resolve, reject) => {
46
- const caller = { texts, options, resolve, reject, done: false };
47
- options?.signal?.addEventListener('abort', () => this.abandon(caller, options.signal.reason), { once: true });
48
- this.queue.push(caller);
49
- this.schedule();
50
- });
51
- }
52
- embedQuery(text, options) {
53
- return this.inner.embedQuery
54
- ? this.inner.embedQuery(text, options)
55
- : this.inner.embed([text], options).then(vectors => vectors[0]);
56
- }
57
- describe() {
58
- return this.inner.describe?.() ?? {};
59
- }
60
- /** A caller that stopped waiting: out of the queue if it has not gone yet, otherwise its request runs on for the others. */
61
- abandon(caller, reason) {
62
- if (caller.done)
63
- return;
64
- caller.done = true;
65
- this.queue = this.queue.filter(c => c !== caller);
66
- caller.reject(reason);
67
- this.schedule();
68
- }
69
- schedule() {
70
- while (this.inFlight < exports.COALESCE_REQUESTS_IN_FLIGHT && this.queue.length > 0) {
71
- const queued = this.queue.reduce((sum, c) => sum + c.texts.length, 0);
72
- if (queued < exports.COALESCE_REQUEST_TEXTS && !this.gatherDone) {
73
- this.gatherTimer ??= setTimeout(() => {
74
- this.gatherTimer = undefined;
75
- this.gatherDone = true;
76
- this.schedule();
77
- }, GATHER_MS);
78
- return;
79
- }
80
- const batch = [];
81
- let size = 0;
82
- // A request is either all interactive (each caller may abort it) or all patient: a patient caller would keep an
83
- // interactive caller's request alive after that caller gave up, and its retries would reach a refusing service.
84
- while (this.queue.length > 0 && (batch.length === 0 || (sameKind(this.queue[0], batch[0])
85
- && size + this.queue[0].texts.length <= exports.COALESCE_REQUEST_TEXTS))) {
86
- const caller = this.queue.shift();
87
- batch.push(caller);
88
- size += caller.texts.length;
89
- }
90
- this.gatherDone = false;
91
- this.inFlight += 1;
92
- this.send(batch);
93
- }
94
- }
95
- send(batch) {
96
- const texts = batch.flatMap(c => c.texts);
97
- const controller = new AbortController();
98
- const budgets = batch.map(c => c.options?.budgetMs);
99
- const budgetMs = budgets.some(b => b === undefined) ? undefined : Math.max(...budgets);
100
- // A caller without a signal waits as long as the service takes: the request then runs to its end.
101
- let waiting = batch.length;
102
- const patient = batch.some(c => c.options?.signal === undefined);
103
- for (const caller of batch) {
104
- const signal = caller.options?.signal;
105
- if (!signal)
106
- continue;
107
- signal.addEventListener('abort', () => {
108
- waiting -= 1;
109
- if (waiting === 0 && !patient)
110
- controller.abort(signal.reason);
111
- }, { once: true });
112
- }
113
- const request = this.inner.embed(texts, { budgetMs, signal: controller.signal });
114
- request.then((vectors) => {
115
- this.inFlight -= 1;
116
- let offset = 0;
117
- for (const caller of batch) {
118
- const own = vectors.slice(offset, offset + caller.texts.length);
119
- offset += caller.texts.length;
120
- if (caller.done)
121
- continue;
122
- caller.done = true;
123
- if (own.length === caller.texts.length)
124
- caller.resolve(own);
125
- else
126
- caller.reject(new Error(`the embedding provider returned ${own.length} vectors for ${caller.texts.length} texts`));
127
- }
128
- this.schedule();
129
- }, (err) => {
130
- this.inFlight -= 1;
131
- this.schedule();
132
- if (isUnavailable(err) || controller.signal.aborted) {
133
- for (const caller of batch) {
134
- if (caller.done)
135
- continue;
136
- caller.done = true;
137
- caller.reject(err);
138
- }
139
- return;
140
- }
141
- // A 400 names an input the service refuses: each caller goes again alone, so the others are not failed by it.
142
- // Any other refusal is the service's answer to the request, and every caller of it gets that answer (no re-sends).
143
- if (!refusesAnInput(err)) {
144
- for (const caller of batch) {
145
- if (caller.done)
146
- continue;
147
- caller.done = true;
148
- caller.reject(err);
149
- }
150
- return;
151
- }
152
- for (const caller of batch) {
153
- if (caller.done)
154
- continue;
155
- this.inner.embed(caller.texts, caller.options).then((vectors) => { if (!caller.done) {
156
- caller.done = true;
157
- caller.resolve(vectors);
158
- } }, (own) => { if (!caller.done) {
159
- caller.done = true;
160
- caller.reject(own);
161
- } });
162
- }
163
- });
164
- }
165
- }
166
- exports.CoalescingEmbeddingProvider = CoalescingEmbeddingProvider;
167
- /** The service did not answer (or the call was cut off): every caller of the request meets the same answer, as each would alone. */
168
- function isUnavailable(err) {
169
- return err instanceof types_1.EmbeddingUnavailableError || (err instanceof Error && err.name === 'AbortError');
170
- }
171
- /** The service refused an input (HTTP 400), not the request as a whole: the callers of the request are sent again one by one. */
172
- function refusesAnInput(err) {
173
- return err instanceof Error && err.status === 400;
174
- }
175
- function sameKind(a, b) {
176
- return (a.options?.signal !== undefined) === (b.options?.signal !== undefined);
177
- }
@@ -1,103 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.InteractiveEmbeddingProvider = void 0;
4
- const types_1 = require("./types");
5
- /**
6
- * An embedding provider for a service an agent waits on.
7
- *
8
- * A call without a budget is passed straight through: patient, the provider's own retries, for a rebuild, a backfill or
9
- * a benchmark. A call with `budgetMs` is interactive: it ends within the budget, with `EmbeddingUnavailableError`, when
10
- * the service is slow, refusing or silent. Google's 429s on 2026-10-06 made `remember` wait up to 13 minutes in the
11
- * provider's retry loop while holding the note's file lock, which every later save and read of that note queued behind.
12
- *
13
- * It also remembers that the service just failed. After an interactive call fails because the service is unavailable,
14
- * interactive calls fail at once for `cooldownMs` without asking it, so a refusing service costs one call a few
15
- * seconds and every other call nothing; the first call after the cooldown is the probe, and one that succeeds (or a
16
- * patient call that does) opens the way again. A failure that is not an outage (a text the service refuses) does not
17
- * close it: it says nothing about the next call.
18
- */
19
- class InteractiveEmbeddingProvider {
20
- dimensions;
21
- identity;
22
- inner;
23
- cooldownMs;
24
- /** Epoch ms before which interactive calls are not made; 0 when the service is believed up. */
25
- unavailableUntil = 0;
26
- /** An interactive call is the probe after a cooldown: the others wait for its verdict instead of piling on. */
27
- probing = false;
28
- constructor(inner, options = {}) {
29
- this.inner = inner;
30
- this.dimensions = inner.dimensions;
31
- this.identity = inner.identity;
32
- this.cooldownMs = options.cooldownMs ?? DEFAULT_COOLDOWN_MS;
33
- }
34
- embed(texts, options) {
35
- return this.run(options, o => this.inner.embed(texts, o));
36
- }
37
- embedQuery(text, options) {
38
- return this.run(options, async (o) => this.inner.embedQuery ? this.inner.embedQuery(text, o) : (await this.inner.embed([text], o))[0]);
39
- }
40
- describe() {
41
- return this.inner.describe?.() ?? {};
42
- }
43
- async run(options, call) {
44
- const budgetMs = options?.budgetMs;
45
- if (budgetMs === undefined) {
46
- // Patient: never refused, and what it learns about the service is passed on.
47
- try {
48
- const result = await call(options);
49
- this.unavailableUntil = 0;
50
- return result;
51
- }
52
- catch (err) {
53
- if (err instanceof types_1.EmbeddingUnavailableError)
54
- this.unavailableUntil = Date.now() + this.cooldownMs;
55
- throw err;
56
- }
57
- }
58
- if (Date.now() < this.unavailableUntil || (this.probing && this.unavailableUntil > 0)) {
59
- throw new types_1.EmbeddingUnavailableError('embedding service unavailable (failed moments ago; not asked again yet)');
60
- }
61
- const probe = this.unavailableUntil > 0;
62
- if (probe)
63
- this.probing = true;
64
- const controller = new AbortController();
65
- const relay = () => controller.abort(options?.signal?.reason);
66
- options?.signal?.addEventListener('abort', relay, { once: true });
67
- let timer;
68
- const expired = new Promise((_, reject) => {
69
- timer = setTimeout(() => {
70
- const unavailable = new types_1.EmbeddingUnavailableError(`embedding service did not answer within ${Math.ceil(budgetMs / 1000)} s`);
71
- // The provider is told to stop (a provider that honours the signal ends its retries and its request); one
72
- // that does not is left to finish unobserved, and its late answer or failure is dropped.
73
- controller.abort(unavailable);
74
- reject(unavailable);
75
- }, budgetMs);
76
- });
77
- try {
78
- const attempt = call({ ...options, signal: controller.signal });
79
- attempt.catch(() => undefined); // an attempt that loses the race settles later; nobody reads it
80
- const result = await Promise.race([attempt, expired]);
81
- this.unavailableUntil = 0;
82
- return result;
83
- }
84
- catch (err) {
85
- // An abort reaches a provider as an AbortError or as the reason it was given; either way it is the budget's.
86
- const unavailable = controller.signal.aborted && controller.signal.reason instanceof types_1.EmbeddingUnavailableError
87
- ? controller.signal.reason
88
- : err;
89
- if (unavailable instanceof types_1.EmbeddingUnavailableError)
90
- this.unavailableUntil = Date.now() + this.cooldownMs;
91
- throw unavailable;
92
- }
93
- finally {
94
- clearTimeout(timer);
95
- options?.signal?.removeEventListener('abort', relay);
96
- if (probe)
97
- this.probing = false;
98
- }
99
- }
100
- }
101
- exports.InteractiveEmbeddingProvider = InteractiveEmbeddingProvider;
102
- /** Long enough that a refusing service is not asked by every call, short enough that a recovered one is used again at once. */
103
- const DEFAULT_COOLDOWN_MS = 15_000;
@@ -1,70 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LocalEmbeddingProvider = void 0;
7
- const path_1 = __importDefault(require("path"));
8
- const os_1 = __importDefault(require("os"));
9
- // Models that require asymmetric query/passage prefixes
10
- const E5_PREFIX_MODELS = ['multilingual-e5', 'e5-'];
11
- function needsE5Prefix(modelName) {
12
- return E5_PREFIX_MODELS.some(p => modelName.toLowerCase().includes(p));
13
- }
14
- // Dimension map for known models
15
- const MODEL_DIMENSIONS = {
16
- 'Xenova/all-MiniLM-L6-v2': 384,
17
- 'Xenova/multilingual-e5-small': 384,
18
- 'Xenova/multilingual-e5-base': 768,
19
- 'Xenova/multilingual-e5-large': 1024,
20
- 'intfloat/multilingual-e5-base': 768,
21
- };
22
- const TRANSFORMERS_PACKAGE = '@xenova/transformers';
23
- class LocalEmbeddingProvider {
24
- dimensions;
25
- identity;
26
- modelName;
27
- extractor = null;
28
- initPromise = null;
29
- constructor(modelName) {
30
- this.modelName = modelName ?? 'Xenova/multilingual-e5-base';
31
- this.dimensions = MODEL_DIMENSIONS[this.modelName] ?? 768;
32
- this.identity = { kind: 'local', model: this.modelName };
33
- }
34
- getExtractor() {
35
- if (this.extractor)
36
- return Promise.resolve(this.extractor);
37
- if (this.initPromise)
38
- return this.initPromise;
39
- this.initPromise = (async () => {
40
- // An optional peer dependency: an `npx renor-engine` install leaves it out (it is hundreds of megabytes), a clone has it.
41
- const { pipeline, env } = await import(TRANSFORMERS_PACKAGE).catch((err) => {
42
- if (!/cannot find (module|package)/i.test(String(err?.message)))
43
- throw err;
44
- throw new Error('EMBEDDING_PROVIDER=local needs the optional package @xenova/transformers: add it (`npx -y -p renor-engine -p @xenova/transformers renor-engine mcp`, or `npm install @xenova/transformers` beside a clone) or set EMBEDDING_PROVIDER=none');
45
- });
46
- env.cacheDir = path_1.default.join(os_1.default.homedir(), '.cache', 'osm-memory', 'models');
47
- const extractor = await pipeline('feature-extraction', this.modelName);
48
- this.extractor = extractor;
49
- return extractor;
50
- })();
51
- return this.initPromise;
52
- }
53
- async embed(texts) {
54
- if (texts.length === 0)
55
- return [];
56
- const extractor = await this.getExtractor();
57
- const prefixed = needsE5Prefix(this.modelName)
58
- ? texts.map(t => `passage: ${t}`)
59
- : texts;
60
- const output = await extractor(prefixed, { pooling: 'mean', normalize: true });
61
- return output.tolist();
62
- }
63
- async embedQuery(text) {
64
- const extractor = await this.getExtractor();
65
- const prefixed = needsE5Prefix(this.modelName) ? `query: ${text}` : text;
66
- const output = await extractor([prefixed], { pooling: 'mean', normalize: true });
67
- return output.tolist()[0];
68
- }
69
- }
70
- exports.LocalEmbeddingProvider = LocalEmbeddingProvider;
@@ -1,106 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.OpenAIEmbeddingProvider = void 0;
4
- /**
5
- * Generic OpenAI-compatible embeddings provider.
6
- * Works with OpenAI, Voyage AI (voyageai.com, OpenAI-shaped /v1/embeddings),
7
- * and any compatible gateway. Configured via EMBEDDING_BASE_URL and
8
- * EMBEDDING_DIMS (defaults preserve upstream OpenAI behavior).
9
- *
10
- * The `openai` SDK is required in the constructor rather than imported: it costs
11
- * ~70 ms to load, it is reached on every cold start of dist/mvp/cli.js through
12
- * this module, and a workspace only ever constructs this class when its
13
- * embedding provider IS openai. A workspace on Vertex (every provisioned
14
- * workspace) or on no provider never constructs it, so for those the SDK was
15
- * ~70 ms of a wake that could not use it. The import above is `import type`, so
16
- * it is erased at compile time and contributes nothing at load — the type stays
17
- * checked, the module is not loaded.
18
- */
19
- class OpenAIEmbeddingProvider {
20
- dimensions;
21
- identity;
22
- /** The dims the user explicitly configured (EMBEDDING_DIMS); null = SDK/gateway default. */
23
- configuredDims;
24
- clients = [];
25
- cooldown = new Map();
26
- rr = 0;
27
- model;
28
- constructor(apiKeys, model, baseURL, dimensions) {
29
- const keys = Array.isArray(apiKeys) ? apiKeys.filter(Boolean) : [apiKeys];
30
- if (keys.length === 0)
31
- throw new Error('at least one API key required');
32
- this.model = model ?? 'text-embedding-3-small';
33
- this.identity = { kind: 'openai', model: this.model };
34
- this.configuredDims = dimensions ?? null;
35
- this.dimensions = dimensions ?? 1536;
36
- // CommonJS loader, deferred: this is the only line that actually pulls the SDK.
37
- const OpenAICtor = require('openai').default;
38
- this.clients = keys.map(apiKey => new OpenAICtor({ apiKey, baseURL }));
39
- }
40
- /** next healthy client index (round-robin over non-cooling keys) */
41
- nextClient() {
42
- const now = Date.now();
43
- for (let t = 0; t < this.clients.length; t++) {
44
- const i = (this.rr + t) % this.clients.length;
45
- if ((this.cooldown.get(i) ?? 0) <= now) {
46
- this.rr = i + 1;
47
- return i;
48
- }
49
- }
50
- return -1; // all cooling
51
- }
52
- async embed(texts) {
53
- const results = [];
54
- const rawBatchSize = process.env.EMBEDDING_BATCH_SIZE;
55
- const batchSize = rawBatchSize === undefined ? 32 : Number(rawBatchSize);
56
- if (!Number.isInteger(batchSize) || batchSize <= 0) {
57
- throw new Error('EMBEDDING_BATCH_SIZE must be a positive integer');
58
- }
59
- for (let i = 0; i < texts.length; i += batchSize) {
60
- const batch = texts.slice(i, i + batchSize);
61
- results.push(...(await this.embedWithRetry(batch)));
62
- }
63
- return results;
64
- }
65
- /** Retries on 429/5xx with exponential backoff; on 429 rotates to the next healthy key first. */
66
- async embedWithRetry(batch, attempt = 0) {
67
- const ci = this.nextClient();
68
- if (ci === -1) {
69
- const waitMs = Math.max(2000, 1000 * 2 ** Math.min(attempt, 8));
70
- await new Promise(r => setTimeout(r, waitMs));
71
- return this.embedWithRetry(batch, attempt + 1);
72
- }
73
- try {
74
- // Product defect fix (P1, disclosed): when EMBEDDING_DIMS is configured it
75
- // sizes the vector index table — it MUST also be sent to the gateway, or
76
- // gateways whose omission default differs (e.g. Google: 3072) return
77
- // mismatched vectors and indexing fails. Unconfigured callers keep the
78
- // previous behavior (SDK/gateway default).
79
- const res = await this.clients[ci].embeddings.create(this.configuredDims !== null
80
- ? { model: this.model, input: batch, dimensions: this.configuredDims }
81
- : { model: this.model, input: batch });
82
- return res.data.map(d => d.embedding);
83
- }
84
- catch (e) {
85
- const err = e;
86
- const status = err?.status;
87
- if (status === 429 && attempt < 40) {
88
- this.cooldown.set(ci, Date.now() + 60_000);
89
- return this.embedWithRetry(batch, attempt + 1); // immediate retry on another key
90
- }
91
- if (status === 401 || status === 403) {
92
- this.cooldown.set(ci, Number.MAX_SAFE_INTEGER); // dead key — never use again this run
93
- if (attempt < 40)
94
- return this.embedWithRetry(batch, attempt + 1);
95
- throw e;
96
- }
97
- if (status !== undefined && status >= 500 && status < 600 && attempt < 10) {
98
- const waitMs = Math.max(Number(err?.headers?.get?.('retry-after')) * 1000 || 0, 1000 * 2 ** attempt, 2000);
99
- await new Promise(r => setTimeout(r, waitMs));
100
- return this.embedWithRetry(batch, attempt + 1);
101
- }
102
- throw e;
103
- }
104
- }
105
- }
106
- exports.OpenAIEmbeddingProvider = OpenAIEmbeddingProvider;
@@ -1,166 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.chunksWithoutVectors = chunksWithoutVectors;
4
- exports.sleep = sleep;
5
- exports.rebuildEmbeddings = rebuildEmbeddings;
6
- exports.embedMissingVectors = embedMissingVectors;
7
- const ATTEMPTS = 3;
8
- /** How many chunks have no stored vector (vec0's rowid shadow table holds one row per stored vector). */
9
- function chunksWithoutVectors(db) {
10
- return db.prepare('SELECT COUNT(*) FROM chunks c WHERE NOT EXISTS (SELECT 1 FROM chunk_embeddings_rowids r WHERE r.rowid = c.id)').pluck().get();
11
- }
12
- /** Wait `ms`, or less if `signal` aborts. The timer never keeps the process alive. */
13
- function sleep(ms, signal) {
14
- return new Promise((resolve) => {
15
- if (ms <= 0 || signal?.aborted)
16
- return resolve();
17
- const done = () => {
18
- clearTimeout(timer);
19
- signal?.removeEventListener('abort', done);
20
- resolve();
21
- };
22
- const timer = setTimeout(done, ms);
23
- timer.unref();
24
- signal?.addEventListener('abort', done, { once: true });
25
- });
26
- }
27
- /**
28
- * Rebuild (or carry on rebuilding) the vectors of `vectors` in `space`, the space `provider` writes. Returns when the
29
- * index is whole, or when `options.signal` aborts. The index is marked whole only when no chunk is left without a
30
- * vector: a chunk the provider will not embed (a text it refuses, or a provider that is down) makes this throw, after
31
- * every other chunk is embedded, and the index stays unfinished for a later call to resume, so semantic search is
32
- * never turned on over an index with a vector missing. A save that changes chunks meanwhile is safe: a vector is
33
- * stored only for a chunk that is still as it was when it was read, and the save embeds its own.
34
- */
35
- async function rebuildEmbeddings(db, vectors, provider, space, options = {}) {
36
- const { batchSize = 32, concurrency = 4, signal, retryDelayMs = (attempt) => 1_000 * 4 ** attempt, } = options;
37
- // An index in another space, or in none that anyone can vouch for, starts over; one in this space that is
38
- // unfinished carries on; one that is whole has nothing to do.
39
- if (vectors.space() !== space)
40
- vectors.beginRebuild(space);
41
- else if (!vectors.rebuilding())
42
- return { done: true, embedded: 0 };
43
- // The chunks that still have no vector. Nothing is ever passed over: a chunk is stored, or it is gone (a save
44
- // replaced it), or this call throws; so the scan is over exactly when no chunk lacks a vector.
45
- const nextRows = db.prepare(`SELECT c.id, c.text FROM chunks c
46
- WHERE NOT EXISTS (SELECT 1 FROM chunk_embeddings_rowids r WHERE r.rowid = c.id)
47
- ORDER BY c.id LIMIT ?`);
48
- const stillThere = db.prepare('SELECT 1 FROM chunks WHERE id = ? AND text = ?');
49
- const noteOf = db.prepare('SELECT note_path FROM chunks WHERE id = ?').pluck();
50
- // Vectors whose chunk is gone: a process with no embedding provider replaces chunks and cannot delete their vectors.
51
- const orphans = db.prepare('SELECT r.rowid FROM chunk_embeddings_rowids r WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.id = r.rowid)').pluck();
52
- const result = { done: false, embedded: 0 };
53
- /** One request: embed these chunks, and keep the vector of each that is still as it was read. */
54
- const embedAndStore = async (rows) => {
55
- const embeddings = await provider.embed(rows.map((r) => r.text));
56
- if (embeddings.length !== rows.length) {
57
- throw new Error(`the embedding provider returned ${embeddings.length} vectors for ${rows.length} texts`);
58
- }
59
- db.transaction(() => {
60
- rows.forEach((row, i) => {
61
- // A save may have replaced this chunk meanwhile; it embedded its own.
62
- if (stillThere.get(row.id, row.text))
63
- vectors.upsert(row.id, embeddings[i]);
64
- });
65
- })();
66
- result.embedded += rows.length;
67
- };
68
- /** Try to embed `rows` (up to ATTEMPTS times); the last error, or null once they are stored. */
69
- const attempt = async (rows) => {
70
- let failure = null;
71
- for (let tries = 0; tries < ATTEMPTS && !signal?.aborted; tries++) {
72
- try {
73
- await embedAndStore(rows);
74
- return null;
75
- }
76
- catch (err) {
77
- failure = err;
78
- if (tries + 1 < ATTEMPTS)
79
- await sleep(retryDelayMs(tries), signal);
80
- }
81
- }
82
- return failure;
83
- };
84
- /**
85
- * Embed `rows`. When a batch keeps failing, embed its chunks one by one, so that every chunk the provider takes is
86
- * stored and only the ones it refuses stay behind; then throw, naming the first. Nothing is skipped.
87
- */
88
- const settle = async (rows) => {
89
- const failure = await attempt(rows);
90
- if (failure === null || signal?.aborted)
91
- return;
92
- let refused;
93
- if (rows.length === 1)
94
- refused = { row: rows[0], failure };
95
- else {
96
- for (const row of rows) {
97
- const one = await attempt([row]);
98
- if (signal?.aborted)
99
- return;
100
- if (one !== null)
101
- refused ??= { row, failure: one };
102
- }
103
- }
104
- if (!refused)
105
- return;
106
- const note = noteOf.get(refused.row.id);
107
- const why = refused.failure instanceof Error ? refused.failure.message : String(refused.failure);
108
- throw Object.assign(new Error(`chunk ${refused.row.id}${note ? ` of ${note}` : ''} could not be embedded: ${why}`), { cause: refused.failure });
109
- };
110
- while (!signal?.aborted) {
111
- const rows = nextRows.all(batchSize * concurrency);
112
- if (rows.length === 0) {
113
- // Whole: every chunk has its vector, and the vectors of chunks that are gone are dropped with the mark.
114
- db.transaction(() => {
115
- for (const id of orphans.all())
116
- vectors.delete(id);
117
- vectors.finishRebuild();
118
- })();
119
- return { ...result, done: true };
120
- }
121
- const groups = [];
122
- for (let i = 0; i < rows.length; i += batchSize)
123
- groups.push(rows.slice(i, i + batchSize));
124
- // Every request settles before this call returns or throws, so nothing is left running against the database.
125
- const settled = await Promise.allSettled(groups.map(settle));
126
- const failed = settled.find((s) => s.status === 'rejected');
127
- if (failed)
128
- throw failed.reason;
129
- }
130
- return result;
131
- }
132
- /**
133
- * Embed every chunk that has no vector, a batch at a time, until none is left or `signal` aborts: the chunks a save
134
- * kept for keyword search because the embedding service would not answer it in time. It is the patient half of an
135
- * interactive save: it waits as long as the provider's own retries take, and holds no lock while it does, so a service
136
- * that is down costs nothing but this task's time. A vector is stored only for a chunk that is still as it was read (a
137
- * save may have replaced it meanwhile and embedded its own). Throws what the provider throws, after which a later call
138
- * carries on with what is left; it does not change what the index says of its own completeness (a rebuild does that).
139
- * Returns how many chunks it embedded.
140
- */
141
- async function embedMissingVectors(db, vectors, provider, options = {}) {
142
- const { batchSize = 32, signal } = options;
143
- const nextRows = db.prepare(`SELECT c.id, c.text FROM chunks c
144
- WHERE NOT EXISTS (SELECT 1 FROM chunk_embeddings_rowids r WHERE r.rowid = c.id)
145
- ORDER BY c.id LIMIT ?`);
146
- const stillThere = db.prepare('SELECT 1 FROM chunks WHERE id = ? AND text = ?');
147
- const embedOptions = signal ? { signal } : undefined;
148
- let embedded = 0;
149
- while (!signal?.aborted) {
150
- const rows = nextRows.all(batchSize);
151
- if (rows.length === 0)
152
- break;
153
- const embeddings = await provider.embed(rows.map((r) => r.text), embedOptions);
154
- if (embeddings.length !== rows.length) {
155
- throw new Error(`the embedding provider returned ${embeddings.length} vectors for ${rows.length} texts`);
156
- }
157
- db.transaction(() => {
158
- rows.forEach((row, i) => {
159
- if (stillThere.get(row.id, row.text))
160
- vectors.upsert(row.id, embeddings[i]);
161
- });
162
- })();
163
- embedded += rows.length;
164
- }
165
- return embedded;
166
- }