renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,597 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LEADING_SUMMARY_REASONS = exports.DIGEST_PRICE_OUT_DEFAULT = exports.DIGEST_PRICE_IN_DEFAULT = exports.DIGEST_MAX_USD_PER_DAY_DEFAULT = exports.DIGEST_HIT_PATH = exports.DIGEST_MAX_WINDOW_NOTES = exports.DIGEST_DEFAULT_TOKENS = exports.DIGEST_RETRIES_DEFAULT = exports.DIGEST_TIMEOUT_MS = exports.DIGEST_MAX_OUTPUT_TOKENS = exports.DIGEST_NOTE_MAX_CHARS = exports.DIGEST_ASSISTANT_TURN_MAX_CHARS = exports.DIGEST_USER_TURN_MAX_CHARS = exports.DIGEST_BATCH_MAX_CHARS = exports.DIGEST_BATCH_MAX_NOTES = exports.DIGEST_INPUT_VERSION = exports.DIGEST_PROMPT_TEMPLATE = void 0;
7
- exports.digestsEnabled = digestsEnabled;
8
- exports.resolveDigestProvider = resolveDigestProvider;
9
- exports.resolveDigestOptions = resolveDigestOptions;
10
- exports.digestInputOf = digestInputOf;
11
- exports.renderDigestPrompt = renderDigestPrompt;
12
- exports.parseDigestReply = parseDigestReply;
13
- exports.digestItemText = digestItemText;
14
- exports.buildNoteDigests = buildNoteDigests;
15
- exports.digestIndexedNotes = digestIndexedNotes;
16
- exports.digestTable = digestTable;
17
- exports.digestItemsToHit = digestItemsToHit;
18
- exports.attachDigest = attachDigest;
19
- exports.leadingSummaryHits = leadingSummaryHits;
20
- const promises_1 = __importDefault(require("node:fs/promises"));
21
- const node_path_1 = __importDefault(require("node:path"));
22
- const o200k_1 = require("../indexer/o200k");
23
- const parser_1 = require("../indexer/parser");
24
- const dailyEntries_1 = require("../indexer/dailyEntries");
25
- const ollamaChat_1 = require("../evidence/ollamaChat");
26
- const keyPool_1 = require("../evidence/keyPool");
27
- const secondPass_1 = require("../evidence/secondPass");
28
- const attach_1 = require("../evidence/attach");
29
- const space_1 = require("../embeddings/space");
30
- const sessionRouter_1 = require("../router/sessionRouter");
31
- const orchestrator_1 = require("../retrieval/orchestrator");
32
- const digestCache_1 = require("./digestCache");
33
- /**
34
- * Write-time digests (RENOR_DIGESTS, default OFF): one model call when a note is indexed writes a query-independent
35
- * list of what the user states in it, with its date and where it was said; search reads that list with no model call.
36
- *
37
- * index (write time) `renor index` / `renor rebuild` with the flag on: one model call per batch of up to
38
- * DIGEST_BATCH_MAX_NOTES notes writes the digest of each note into the digest cache, keyed by
39
- * the note's file hash, so an unchanged note is never paid for twice. Each item is then embedded
40
- * once with the tenant's embedding provider and the vectors are stored beside it.
41
- * search (read time) no model call. The digests of the notes the retrieval window already holds are scored by
42
- * cosine to the query embedding, the best fit DIGEST_DEFAULT_TOKENS tokens, ordered by date,
43
- * and prepended as one hit rendered like the evidence table.
44
- *
45
- * Retrieval, ranking and stored chunk text are never changed. Flag off: nothing here runs, no cache file is created,
46
- * and `search --json` carries no `digest` field.
47
- *
48
- * Provenance: every item keeps the turn or section it was said in (`source`), and the search report lists, per
49
- * rendered item, the note and that source. The statement itself stays in the note.
50
- *
51
- * The digest reads the user's own turns in full and clips the assistant's turns, since the list is of what the user
52
- * states; a note with no turn structure is read whole (capped). Notes with nothing to say get an empty digest.
53
- */
54
- exports.DIGEST_PROMPT_TEMPLATE = `You write a digest of a person's past conversations for their long-term memory. Below are {n} conversations between the person ("User") and an AI assistant. Each starts with an id and a date; turns are labelled [Turn N User] or [Turn N Assistant] (assistant turns may be shortened).
55
-
56
- TASK: For EACH conversation list EVERY distinct item, event, amount, preference or change of state that the User states about their own life or situation. Rules:
57
- - One entry per distinct item (e.g., each product, each trip, each device, each course, each amount, each plan, each preference).
58
- - "item" = what it is, specific enough to be understood without the conversation.
59
- - "action" = what happened or holds (bought / visited / completed / raised / inherited / prefers / plans / changed to / ...).
60
- - "amount" = numeric value + unit when stated (e.g., "3 days", "$120", "100 points"); else "".
61
- - "date" = the date the item happened or applies when the User gives one (resolve "yesterday", "last week" against the conversation's date); else the conversation's date.
62
- - "turn" = the label of the turn where the User stated it, e.g. "Turn 3".
63
- - Do NOT compute any totals. Do NOT merge different items. Include every candidate even if uncertain.
64
- - Skip general knowledge and advice given by the assistant. If the User states nothing about their own life or situation, return an empty list for that conversation.
65
-
66
- {conversations}
67
-
68
- Respond with ONLY minified JSON, no markdown, one entry per conversation using the ids given:
69
- {{"notes":[{{"id":"<id>","items":[{{"item":"...","action":"...","amount":"...","date":"...","turn":"..."}}]}}]}}`;
70
- /** Part of every digest's cache key: change how a note is shown to the model and every note is digested again. */
71
- exports.DIGEST_INPUT_VERSION = 'user-turns-v1';
72
- /** A digest call carries at most this many notes or characters of them. */
73
- exports.DIGEST_BATCH_MAX_NOTES = 6;
74
- exports.DIGEST_BATCH_MAX_CHARS = 40_000;
75
- /** The user's turn is shown up to this many characters, the assistant's turn up to the second number. */
76
- exports.DIGEST_USER_TURN_MAX_CHARS = 4_000;
77
- exports.DIGEST_ASSISTANT_TURN_MAX_CHARS = 300;
78
- /** A note is shown at most this long (its head, then its tail, when longer). */
79
- exports.DIGEST_NOTE_MAX_CHARS = 20_000;
80
- exports.DIGEST_MAX_OUTPUT_TOKENS = 8_000;
81
- exports.DIGEST_TIMEOUT_MS = 180_000;
82
- exports.DIGEST_RETRIES_DEFAULT = 3;
83
- const DIGEST_ATTEMPTS = 2;
84
- const DIGEST_MAX_ITEMS_PER_NOTE = 40;
85
- /** What the rendered table may weigh at search time, in gpt-tokenizer tokens (table header and footer included). */
86
- exports.DIGEST_DEFAULT_TOKENS = 600;
87
- /** At most this many of the retrieval window's notes are read, like the evidence pass's 20 sessions. */
88
- exports.DIGEST_MAX_WINDOW_NOTES = 20;
89
- exports.DIGEST_HIT_PATH = '_renor/digest';
90
- function digestsEnabled(env = process.env) {
91
- const raw = (env.RENOR_DIGESTS ?? '').trim().toLowerCase();
92
- if (raw === '' || raw === '0' || raw === 'off' || raw === 'false' || raw === 'no')
93
- return false;
94
- if (raw === '1' || raw === 'on' || raw === 'true' || raw === 'yes')
95
- return true;
96
- // A typo must not quietly measure the control arm twice.
97
- throw new Error(`Invalid RENOR_DIGESTS "${raw}": must be on | off`);
98
- }
99
- exports.DIGEST_MAX_USD_PER_DAY_DEFAULT = 0.5;
100
- exports.DIGEST_PRICE_IN_DEFAULT = 0.225;
101
- exports.DIGEST_PRICE_OUT_DEFAULT = 0.75;
102
- function positiveNumber(raw, fallback, name) {
103
- const text = (raw ?? '').trim();
104
- if (text === '')
105
- return fallback;
106
- const value = Number(text);
107
- // A typo must not quietly lift or remove a cost guard.
108
- if (!Number.isFinite(value) || value <= 0)
109
- throw new Error(`Invalid ${name} "${text}": must be a positive number`);
110
- return value;
111
- }
112
- function boundedInt(raw, fallback, min, max) {
113
- const value = Number((raw ?? '').trim());
114
- if ((raw ?? '').trim() === '' || !Number.isFinite(value))
115
- return fallback;
116
- return Math.min(max, Math.max(min, Math.floor(value)));
117
- }
118
- /**
119
- * The digest model's provider: the digest slot (RENOR_DIGEST_BASE_URL + RENOR_DIGEST_API_KEY(S)) and nothing else.
120
- *
121
- * A digest call sends the text of notes (clipped, see DIGEST_NOTE_MAX_CHARS), and the Privacy Policy, the LGPD page and
122
- * the register of processing (ROPA P13) name one processor for it. So this never borrows the evidence provider's key or
123
- * URL, the session router's, or an Ollama Cloud key, and the transport's default URL (Ollama Cloud) is never reached:
124
- * with the slot unset no client is built and nothing is sent. Pointing the slot at a provider those pages do not name
125
- * means updating them first.
126
- */
127
- function resolveDigestProvider(env = process.env) {
128
- const keys = [...(0, keyPool_1.splitKeyList)(env.RENOR_DIGEST_API_KEYS), ...(0, keyPool_1.splitKeyList)(env.RENOR_DIGEST_API_KEY)];
129
- return { baseUrl: env.RENOR_DIGEST_BASE_URL?.trim() || undefined, keys };
130
- }
131
- /** Env -> options, or null when the flag is off. */
132
- function resolveDigestOptions(config, env = process.env) {
133
- if (!digestsEnabled(env))
134
- return null;
135
- const effortRaw = (env.RENOR_DIGEST_REASONING_EFFORT ?? 'low').trim().toLowerCase();
136
- if (!secondPass_1.EVIDENCE_REASONING_EFFORTS.includes(effortRaw)) {
137
- throw new Error(`Invalid RENOR_DIGEST_REASONING_EFFORT "${effortRaw}": must be ${secondPass_1.EVIDENCE_REASONING_EFFORTS.join(' | ')}`);
138
- }
139
- const cacheOnlyRaw = (env.RENOR_DIGEST_CACHE_ONLY ?? '').trim().toLowerCase();
140
- const cacheOnly = cacheOnlyRaw === '1' || cacheOnlyRaw === 'on' || cacheOnlyRaw === 'true' || cacheOnlyRaw === 'yes';
141
- const retries = boundedInt(env.RENOR_DIGEST_RETRIES, exports.DIGEST_RETRIES_DEFAULT, 0, 8);
142
- const { baseUrl, keys } = resolveDigestProvider(env);
143
- return {
144
- model: env.RENOR_DIGEST_MODEL?.trim() || env.RENOR_EVIDENCE_MODEL?.trim() || secondPass_1.EVIDENCE_MODEL_DEFAULT,
145
- effort: effortRaw,
146
- cachePath: env.RENOR_DIGEST_CACHE?.trim() || node_path_1.default.join(node_path_1.default.dirname(config.dbPath), 'digest-cache.db'),
147
- cacheOnly,
148
- tokens: boundedInt(env.RENOR_DIGEST_TOKENS, exports.DIGEST_DEFAULT_TOKENS, 100, 4_000),
149
- retries,
150
- maxUsdPerDay: positiveNumber(env.RENOR_DIGEST_MAX_USD_PER_DAY, exports.DIGEST_MAX_USD_PER_DAY_DEFAULT, 'RENOR_DIGEST_MAX_USD_PER_DAY'),
151
- priceInPerM: positiveNumber(env.RENOR_DIGEST_PRICE_IN_PER_M, exports.DIGEST_PRICE_IN_DEFAULT, 'RENOR_DIGEST_PRICE_IN_PER_M'),
152
- priceOutPerM: positiveNumber(env.RENOR_DIGEST_PRICE_OUT_PER_M, exports.DIGEST_PRICE_OUT_DEFAULT, 'RENOR_DIGEST_PRICE_OUT_PER_M'),
153
- // A client needs the slot's key AND its URL: a key alone would send notes to the transport's default endpoint.
154
- ...(keys.length > 0 && baseUrl && !cacheOnly ? { chat: (0, ollamaChat_1.createOllamaChat)({ baseUrl, keys, timeoutMs: exports.DIGEST_TIMEOUT_MS, maxRetries: retries }) } : {}),
155
- };
156
- }
157
- // ─── What the model is shown ───
158
- const TURN_HEADING = /^### Turn (\d+) — ([^\n]*?)[ \t]*$/gm;
159
- /**
160
- * The text a note is digested from. A note made of `### Turn N — Role` sections shows the user's turns whole (up to a
161
- * cap) and the assistant's turns clipped; any other note is shown whole, or its head and tail when long.
162
- */
163
- function digestInputOf(body) {
164
- const matches = [...body.matchAll(TURN_HEADING)];
165
- if (matches.length === 0)
166
- return clipLong(body);
167
- const parts = [];
168
- matches.forEach((match, index) => {
169
- const start = (match.index ?? 0) + match[0].length;
170
- const end = index + 1 < matches.length ? (matches[index + 1].index ?? body.length) : body.length;
171
- const text = body.slice(start, end).trim();
172
- const role = match[2].trim();
173
- const assistant = /^assistant$/i.test(role);
174
- const cap = assistant ? exports.DIGEST_ASSISTANT_TURN_MAX_CHARS : exports.DIGEST_USER_TURN_MAX_CHARS;
175
- const shown = text.length > cap ? `${text.slice(0, cap)} …` : text;
176
- parts.push(`[Turn ${match[1]} ${role}]\n${shown}`);
177
- });
178
- return clipLong(parts.join('\n\n'));
179
- }
180
- function clipLong(text) {
181
- if (text.length <= exports.DIGEST_NOTE_MAX_CHARS)
182
- return text;
183
- return `${text.slice(0, 14_000)}\n\n[... middle omitted ...]\n\n${text.slice(-6_000)}`;
184
- }
185
- function renderDigestPrompt(batch) {
186
- const conversations = batch.map(item => `=== CONVERSATION id=${item.id} date=${item.date} ===\n${item.text}`).join('\n\n');
187
- return exports.DIGEST_PROMPT_TEMPLATE.replace('{n}', () => String(batch.length)).replace('{conversations}', () => conversations);
188
- }
189
- const batchId = (id) => id.trim().toLowerCase().replace(/^c0*(\d+)$/, 'c$1');
190
- const clean = (value, max) => (typeof value === 'string' ? value : typeof value === 'number' ? String(value) : '').replace(/\s+/g, ' ').trim().slice(0, max);
191
- /**
192
- * `{"notes":[{id, items:[{item, action, amount, date, turn}]}]}` -> id (normalised) -> items. An entry whose items
193
- * list is missing is dropped (the note is asked again); an empty list is a real answer. An item without text is dropped.
194
- */
195
- function parseDigestReply(text) {
196
- const out = new Map();
197
- const trimmed = text.trim().replace(/^```(?:json)?\s*|\s*```$/g, '');
198
- let payload;
199
- try {
200
- payload = JSON.parse(trimmed);
201
- }
202
- catch {
203
- const match = trimmed.match(/\{[\s\S]*\}/);
204
- if (!match)
205
- return out;
206
- try {
207
- payload = JSON.parse(match[0]);
208
- }
209
- catch {
210
- return out;
211
- }
212
- }
213
- const notes = payload && typeof payload === 'object' ? payload.notes : undefined;
214
- if (!Array.isArray(notes))
215
- return out;
216
- for (const note of notes) {
217
- if (!note || typeof note !== 'object' || Array.isArray(note))
218
- continue;
219
- const record = note;
220
- const id = batchId(String(record.id ?? ''));
221
- if (!id || !Array.isArray(record.items))
222
- continue;
223
- const items = [];
224
- for (const raw of record.items) {
225
- if (!raw || typeof raw !== 'object' || Array.isArray(raw))
226
- continue;
227
- const r = raw;
228
- const item = clean(r.item, 200);
229
- if (!item)
230
- continue;
231
- items.push({
232
- item,
233
- action: clean(r.action, 60),
234
- amount: clean(r.amount, 60),
235
- date: clean(r.date, 40),
236
- source: clean(r.turn ?? r.source, 30),
237
- });
238
- if (items.length >= DIGEST_MAX_ITEMS_PER_NOTE)
239
- break;
240
- }
241
- out.set(id, items);
242
- }
243
- return out;
244
- }
245
- function batchesOf(items) {
246
- const batches = [];
247
- let current = [];
248
- let chars = 0;
249
- for (const item of items) {
250
- if (current.length > 0 && (chars + item.text.length > exports.DIGEST_BATCH_MAX_CHARS || current.length >= exports.DIGEST_BATCH_MAX_NOTES)) {
251
- batches.push(current);
252
- current = [];
253
- chars = 0;
254
- }
255
- current.push(item);
256
- chars += item.text.length;
257
- }
258
- if (current.length > 0)
259
- batches.push(current);
260
- return batches;
261
- }
262
- /** The text one item is embedded as: the row a reader would see, without the date. */
263
- function digestItemText(item) {
264
- return [item.item, item.action, item.amount].filter(Boolean).join(' | ');
265
- }
266
- /**
267
- * Give every note a digest, calling the model only for notes the cache does not hold, then give every digest the
268
- * vectors of its items in the provider's embedding space. Batches run one after another.
269
- */
270
- async function buildNoteDigests(args) {
271
- const started = Date.now();
272
- const { options, cache } = args;
273
- const model = options.model;
274
- const space = (0, space_1.embeddingSpace)(args.provider);
275
- const all = (0, sessionRouter_1.listRouterNotes)(args.db, args.vaultPath, args.ledgerRelPath);
276
- const notes = args.onlyPaths ? all.filter(note => args.onlyPaths.has(note.path)) : all;
277
- const stats = {
278
- model, notes: notes.length, cached: 0, digested: 0, skippedUncached: 0, failed: 0, gone: 0, calls: 0,
279
- promptTokens: 0, completionTokens: 0, items: 0, embeddedTexts: 0, ms: 0,
280
- };
281
- const keyOf = (noteHash) => digestCache_1.DigestCache.digestKey(model, options.effort, exports.DIGEST_PROMPT_TEMPLATE, exports.DIGEST_INPUT_VERSION, noteHash);
282
- const pending = [];
283
- for (const note of notes) {
284
- const key = keyOf(note.noteHash);
285
- if (cache.getDigest(key)) {
286
- stats.cached++;
287
- continue;
288
- }
289
- if (options.cacheOnly) {
290
- stats.skippedUncached++;
291
- continue;
292
- }
293
- let raw;
294
- try {
295
- raw = await promises_1.default.readFile(note.file, 'utf8');
296
- }
297
- catch (err) {
298
- if (err.code === 'ENOENT') {
299
- stats.gone++;
300
- continue;
301
- }
302
- stats.failed++;
303
- stats.error ??= `unreadable note: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
304
- continue;
305
- }
306
- if ((0, dailyEntries_1.noteContentHash)(note.file, raw) !== note.noteHash) {
307
- // Edited after it was indexed: its digest must describe what search will find, so it waits for the next index.
308
- stats.failed++;
309
- stats.error ??= 'a note changed after it was indexed; index again';
310
- continue;
311
- }
312
- const parsed = (0, parser_1.parseNote)(note.file, raw);
313
- if (!parsed.body.trim()) {
314
- cache.putDigest(key, note.noteHash, { items: [], model });
315
- stats.digested++;
316
- continue;
317
- }
318
- pending.push({ key, noteHash: note.noteHash, date: note.date ?? '', text: digestInputOf(parsed.body) });
319
- }
320
- if (pending.length > 0 && !options.chat) {
321
- stats.failed += pending.length;
322
- stats.error ??= 'no digest model configured';
323
- }
324
- else {
325
- await digestPending(pending, options, cache, stats);
326
- }
327
- await embedDigests(notes, keyOf, cache, args.provider, space, stats);
328
- stats.ms = Date.now() - started;
329
- return stats;
330
- }
331
- async function digestPending(pending, options, cache, stats) {
332
- const batches = batchesOf(pending);
333
- for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
334
- const items = batches[batchIndex].map((entry, index) => ({ ...entry, id: `c${String(index + 1).padStart(2, '0')}` }));
335
- let prompt = renderDigestPrompt(items);
336
- const left = new Map(items.map(item => [batchId(item.id), item]));
337
- let transportFailed = false;
338
- for (let attempt = 0; attempt < DIGEST_ATTEMPTS && left.size > 0; attempt++) {
339
- let content;
340
- try {
341
- const res = await options.chat({
342
- model: options.model,
343
- messages: [{ role: 'user', content: prompt }],
344
- temperature: 0,
345
- response_format: { type: 'json_object' },
346
- max_tokens: exports.DIGEST_MAX_OUTPUT_TOKENS,
347
- reasoning_effort: options.effort,
348
- });
349
- stats.calls++;
350
- stats.promptTokens += res.promptTokens ?? 0;
351
- stats.completionTokens += (0, sessionRouter_1.billedOutputTokens)(res);
352
- content = res.content;
353
- }
354
- catch (err) {
355
- stats.calls++;
356
- stats.error ??= `digest call failed: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
357
- transportFailed = true;
358
- break;
359
- }
360
- const parsed = parseDigestReply(content);
361
- for (const [id, item] of [...left]) {
362
- const found = parsed.get(id);
363
- if (!found)
364
- continue;
365
- const dated = found.map(it => (it.date ? it : { ...it, date: item.date }));
366
- cache.putDigest(item.key, item.noteHash, { items: dated, model: options.model });
367
- stats.digested++;
368
- left.delete(id);
369
- }
370
- // A different request, so a provider-side cache cannot replay the incomplete answer.
371
- prompt += '\n';
372
- }
373
- if (left.size > 0) {
374
- stats.failed += left.size;
375
- stats.error ??= 'the digest model left some notes without a digest';
376
- }
377
- if (transportFailed) {
378
- // The transport already retried: a provider that is down or out of quota fails the next batch the same way.
379
- for (const rest of batches.slice(batchIndex + 1))
380
- stats.failed += rest.length;
381
- break;
382
- }
383
- }
384
- }
385
- const EMBED_BATCH = 64;
386
- async function embedDigests(notes, keyOf, cache, provider, space, stats) {
387
- if (space === null)
388
- return;
389
- const waiting = [];
390
- for (const note of notes) {
391
- const key = keyOf(note.noteHash);
392
- const digest = cache.getDigest(key);
393
- if (!digest || digest.items.length === 0)
394
- continue;
395
- stats.items += digest.items.length;
396
- if (cache.getVectors(key, space, digest.items.length))
397
- continue;
398
- waiting.push({ key, texts: digest.items.map(digestItemText) });
399
- }
400
- for (let i = 0; i < waiting.length; i += EMBED_BATCH) {
401
- const group = waiting.slice(i, i + EMBED_BATCH);
402
- const texts = group.flatMap(entry => entry.texts);
403
- let vectors;
404
- try {
405
- vectors = await provider.embed(texts);
406
- }
407
- catch (err) {
408
- stats.failed += group.length;
409
- stats.error ??= `digest embedding failed: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
410
- return;
411
- }
412
- stats.embeddedTexts += texts.length;
413
- let at = 0;
414
- for (const entry of group) {
415
- cache.putVectors(entry.key, space, vectors.slice(at, at + entry.texts.length));
416
- at += entry.texts.length;
417
- }
418
- }
419
- }
420
- /**
421
- * `renor index` / `renor rebuild`: build the digests. The caller opens and closes nothing. Not under the daily cost guard
422
- * (the background worker's, worker.ts): this is a command an operator runs, and the `[digests]` line it prints says what
423
- * it spent. It must not be run from an unattended job with RENOR_DIGESTS on.
424
- */
425
- async function digestIndexedNotes(args) {
426
- const cache = new digestCache_1.DigestCache(args.options.cachePath);
427
- try {
428
- return await buildNoteDigests({
429
- db: args.db,
430
- vaultPath: args.config.vaultPath,
431
- ledgerRelPath: args.config.memoryLedger,
432
- options: args.options,
433
- cache,
434
- provider: args.provider,
435
- onlyPaths: args.onlyPaths,
436
- });
437
- }
438
- finally {
439
- cache.close();
440
- }
441
- }
442
- function cosine(a, b) {
443
- let dot = 0;
444
- let na = 0;
445
- let nb = 0;
446
- const n = Math.min(a.length, b.length);
447
- for (let i = 0; i < n; i++) {
448
- dot += a[i] * b[i];
449
- na += a[i] * a[i];
450
- nb += b[i] * b[i];
451
- }
452
- return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb);
453
- }
454
- function rowTokens(item) {
455
- const action = item.action?.trim() ? ` | ${item.action.trim()}` : '';
456
- const amount = item.amount?.trim() ? ` | ${item.amount.trim()}` : '';
457
- const date = item.date?.trim() ? ` | ${item.date.trim()}` : '';
458
- return (0, o200k_1.countO200k)(`- ${item.item}${action}${amount}${date}\n`);
459
- }
460
- /**
461
- * The table of digest items for one search, and nothing else: the digests of the window's notes (first
462
- * DIGEST_MAX_WINDOW_NOTES distinct notes of `hits`, never the ledger or a synthetic hit), the best-fitting items by
463
- * cosine to the query, as many as fit `options.tokens`, ordered oldest first. Reads the cache and the query
464
- * embedding only.
465
- */
466
- async function digestTable(args) {
467
- const report = {
468
- status: 'skipped', windowNotes: 0, notesWithDigest: 0, itemsConsidered: 0, itemsRendered: 0, tokens: 0, text: '', items: [],
469
- };
470
- const paths = [];
471
- for (const hit of args.hits) {
472
- if (!hit.notePath || hit.notePath.startsWith('_renor/') || paths.includes(hit.notePath))
473
- continue;
474
- if (args.isLedgerNote?.(hit.notePath))
475
- continue;
476
- paths.push(hit.notePath);
477
- if (paths.length >= exports.DIGEST_MAX_WINDOW_NOTES)
478
- break;
479
- }
480
- report.windowNotes = paths.length;
481
- if (paths.length === 0)
482
- return { items: [], report: { ...report, reason: 'no-notes' } };
483
- const space = (0, space_1.embeddingSpace)(args.provider);
484
- if (space === null)
485
- return { items: [], report: { ...report, reason: 'no-embedding-space' } };
486
- let cache;
487
- try {
488
- cache = new digestCache_1.DigestCache(args.options.cachePath, { readonly: true });
489
- }
490
- catch (err) {
491
- // No cache file: nothing was ever digested here, which is the same as no digests.
492
- return { items: [], report: { ...report, reason: 'no-digests', error: `digest cache unavailable: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200) } };
493
- }
494
- try {
495
- const noteRow = args.db.prepare('SELECT note_hash, frontmatter_json, modified_at FROM notes WHERE path = ?');
496
- const candidates = [];
497
- let missingVectors = 0;
498
- paths.forEach((notePath, order) => {
499
- const row = noteRow.get(notePath);
500
- if (!row || !row.note_hash)
501
- return;
502
- const key = digestCache_1.DigestCache.digestKey(args.options.model, args.options.effort, exports.DIGEST_PROMPT_TEMPLATE, exports.DIGEST_INPUT_VERSION, row.note_hash);
503
- const digest = cache.getDigest(key);
504
- if (!digest)
505
- return;
506
- report.notesWithDigest++;
507
- if (digest.items.length === 0)
508
- return;
509
- const vectors = cache.getVectors(key, space, digest.items.length);
510
- if (!vectors) {
511
- missingVectors++;
512
- return;
513
- }
514
- const noteDate = (0, orchestrator_1.resolveSessionDate)(row.frontmatter_json, row.modified_at);
515
- digest.items.forEach((item, index) => {
516
- // A date the model wrote in words ("last Tuesday") does not order; the note's own date does.
517
- const own = (0, orchestrator_1.sessionDateMs)(item.date);
518
- const dateMs = own === Number.NEGATIVE_INFINITY ? (0, orchestrator_1.sessionDateMs)(noteDate) : own;
519
- candidates.push({ notePath, order, index, item, vector: vectors[index], dateMs });
520
- });
521
- });
522
- report.itemsConsidered = candidates.length;
523
- if (candidates.length === 0) {
524
- return { items: [], report: { ...report, reason: report.notesWithDigest === 0 ? 'no-digests' : missingVectors > 0 ? 'no-vectors' : 'no-digests' } };
525
- }
526
- const queryVector = args.queryVector
527
- ?? (args.provider.embedQuery
528
- ? await args.provider.embedQuery(args.query)
529
- : (await args.provider.embed([args.query]))[0]);
530
- const ranked = candidates
531
- .map(candidate => ({ candidate, score: cosine(queryVector, candidate.vector) }))
532
- .sort((a, b) => b.score - a.score || a.candidate.order - b.candidate.order || a.candidate.index - b.candidate.index);
533
- const frame = (0, o200k_1.countO200k)((0, attach_1.renderEvidenceTable)([]));
534
- let used = frame;
535
- const chosen = [];
536
- for (const { candidate } of ranked) {
537
- const cost = rowTokens(candidate.item);
538
- if (used + cost > args.options.tokens)
539
- break;
540
- used += cost;
541
- chosen.push(candidate);
542
- }
543
- chosen.sort((a, b) => (a.dateMs === b.dateMs ? 0 : a.dateMs < b.dateMs ? -1 : 1) || a.order - b.order || a.index - b.index);
544
- const items = chosen.map(c => ({ item: c.item.item, action: c.item.action, amount: c.item.amount, date: c.item.date }));
545
- report.status = items.length > 0 ? 'ok' : 'skipped';
546
- report.itemsRendered = items.length;
547
- report.text = items.length > 0 ? (0, attach_1.renderEvidenceTable)(items) : '';
548
- report.tokens = items.length > 0 ? (0, o200k_1.countO200k)(report.text) : 0;
549
- report.items = chosen.map(c => ({ notePath: c.notePath, source: c.item.source, item: c.item.item }));
550
- return { items, report };
551
- }
552
- finally {
553
- cache.close();
554
- }
555
- }
556
- /** The digest as one hit, rendered like the evidence table (the same rows the reader sees from the evidence pass). */
557
- function digestItemsToHit(items) {
558
- return {
559
- notePath: exports.DIGEST_HIT_PATH,
560
- chunkId: -1,
561
- headingPath: 'Digest',
562
- text: (0, attach_1.renderEvidenceTable)(items),
563
- score: 1,
564
- reason: 'digest',
565
- };
566
- }
567
- /** `renor search`: the hits with the digest hit placed after any synthetic hits already leading, and the report. */
568
- async function attachDigest(args) {
569
- try {
570
- const { items, report } = await digestTable(args);
571
- if (items.length === 0)
572
- return { hits: args.hits, report };
573
- let at = 0;
574
- while (at < args.hits.length && args.hits[at].notePath.startsWith('_renor/'))
575
- at++;
576
- return { hits: [...args.hits.slice(0, at), digestItemsToHit(items), ...args.hits.slice(at)], report };
577
- }
578
- catch (err) {
579
- // Optional augmentation: a fault here must never take down the base search, and it is said, not left silent.
580
- return {
581
- hits: args.hits,
582
- report: {
583
- status: 'skipped', reason: 'digest-error', windowNotes: 0, notesWithDigest: 0, itemsConsidered: 0,
584
- itemsRendered: 0, tokens: 0, text: '', items: [], error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
585
- },
586
- };
587
- }
588
- }
589
- /** The `reason` of the synthetic hits that lead a search page: the evidence table and the digest table. */
590
- exports.LEADING_SUMMARY_REASONS = ['evidence', 'digest'];
591
- /** How many hits at the head of a page are synthetic summary tables (they sit outside top_k and the ranking). */
592
- function leadingSummaryHits(hits) {
593
- let n = 0;
594
- while (n < hits.length && exports.LEADING_SUMMARY_REASONS.includes(hits[n].reason ?? ''))
595
- n++;
596
- return n;
597
- }