renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,597 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LEADING_SUMMARY_REASONS = exports.DIGEST_PRICE_OUT_DEFAULT = exports.DIGEST_PRICE_IN_DEFAULT = exports.DIGEST_MAX_USD_PER_DAY_DEFAULT = exports.DIGEST_HIT_PATH = exports.DIGEST_MAX_WINDOW_NOTES = exports.DIGEST_DEFAULT_TOKENS = exports.DIGEST_RETRIES_DEFAULT = exports.DIGEST_TIMEOUT_MS = exports.DIGEST_MAX_OUTPUT_TOKENS = exports.DIGEST_NOTE_MAX_CHARS = exports.DIGEST_ASSISTANT_TURN_MAX_CHARS = exports.DIGEST_USER_TURN_MAX_CHARS = exports.DIGEST_BATCH_MAX_CHARS = exports.DIGEST_BATCH_MAX_NOTES = exports.DIGEST_INPUT_VERSION = exports.DIGEST_PROMPT_TEMPLATE = void 0;
7
- exports.digestsEnabled = digestsEnabled;
8
- exports.resolveDigestProvider = resolveDigestProvider;
9
- exports.resolveDigestOptions = resolveDigestOptions;
10
- exports.digestInputOf = digestInputOf;
11
- exports.renderDigestPrompt = renderDigestPrompt;
12
- exports.parseDigestReply = parseDigestReply;
13
- exports.digestItemText = digestItemText;
14
- exports.buildNoteDigests = buildNoteDigests;
15
- exports.digestIndexedNotes = digestIndexedNotes;
16
- exports.digestTable = digestTable;
17
- exports.digestItemsToHit = digestItemsToHit;
18
- exports.attachDigest = attachDigest;
19
- exports.leadingSummaryHits = leadingSummaryHits;
20
- const promises_1 = __importDefault(require("node:fs/promises"));
21
- const node_path_1 = __importDefault(require("node:path"));
22
- const o200k_1 = require("../indexer/o200k");
23
- const parser_1 = require("../indexer/parser");
24
- const dailyEntries_1 = require("../indexer/dailyEntries");
25
- const ollamaChat_1 = require("../evidence/ollamaChat");
26
- const keyPool_1 = require("../evidence/keyPool");
27
- const secondPass_1 = require("../evidence/secondPass");
28
- const attach_1 = require("../evidence/attach");
29
- const space_1 = require("../embeddings/space");
30
- const sessionRouter_1 = require("../router/sessionRouter");
31
- const orchestrator_1 = require("../retrieval/orchestrator");
32
- const digestCache_1 = require("./digestCache");
33
- /**
34
- * Write-time digests (RENOR_DIGESTS, default OFF): one model call when a note is indexed writes a query-independent
35
- * list of what the user states in it, with its date and where it was said; search reads that list with no model call.
36
- *
37
- * index (write time) `renor index` / `renor rebuild` with the flag on: one model call per batch of up to
38
- * DIGEST_BATCH_MAX_NOTES notes writes the digest of each note into the digest cache, keyed by
39
- * the note's file hash, so an unchanged note is never paid for twice. Each item is then embedded
40
- * once with the tenant's embedding provider and the vectors are stored beside it.
41
- * search (read time) no model call. The digests of the notes the retrieval window already holds are scored by
42
- * cosine to the query embedding, the best fit DIGEST_DEFAULT_TOKENS tokens, ordered by date,
43
- * and prepended as one hit rendered like the evidence table.
44
- *
45
- * Retrieval, ranking and stored chunk text are never changed. Flag off: nothing here runs, no cache file is created,
46
- * and `search --json` carries no `digest` field.
47
- *
48
- * Provenance: every item keeps the turn or section it was said in (`source`), and the search report lists, per
49
- * rendered item, the note and that source. The statement itself stays in the note.
50
- *
51
- * The digest reads the user's own turns in full and clips the assistant's turns, since the list is of what the user
52
- * states; a note with no turn structure is read whole (capped). Notes with nothing to say get an empty digest.
53
- */
54
- exports.DIGEST_PROMPT_TEMPLATE = `You write a digest of a person's past conversations for their long-term memory. Below are {n} conversations between the person ("User") and an AI assistant. Each starts with an id and a date; turns are labelled [Turn N User] or [Turn N Assistant] (assistant turns may be shortened).
55
-
56
- TASK: For EACH conversation list EVERY distinct item, event, amount, preference or change of state that the User states about their own life or situation. Rules:
57
- - One entry per distinct item (e.g., each product, each trip, each device, each course, each amount, each plan, each preference).
58
- - "item" = what it is, specific enough to be understood without the conversation.
59
- - "action" = what happened or holds (bought / visited / completed / raised / inherited / prefers / plans / changed to / ...).
60
- - "amount" = numeric value + unit when stated (e.g., "3 days", "$120", "100 points"); else "".
61
- - "date" = the date the item happened or applies when the User gives one (resolve "yesterday", "last week" against the conversation's date); else the conversation's date.
62
- - "turn" = the label of the turn where the User stated it, e.g. "Turn 3".
63
- - Do NOT compute any totals. Do NOT merge different items. Include every candidate even if uncertain.
64
- - Skip general knowledge and advice given by the assistant. If the User states nothing about their own life or situation, return an empty list for that conversation.
65
-
66
- {conversations}
67
-
68
- Respond with ONLY minified JSON, no markdown, one entry per conversation using the ids given:
69
- {{"notes":[{{"id":"<id>","items":[{{"item":"...","action":"...","amount":"...","date":"...","turn":"..."}}]}}]}}`;
70
- /** Part of every digest's cache key: change how a note is shown to the model and every note is digested again. */
71
- exports.DIGEST_INPUT_VERSION = 'user-turns-v1';
72
- /** A digest call carries at most this many notes or characters of them. */
73
- exports.DIGEST_BATCH_MAX_NOTES = 6;
74
- exports.DIGEST_BATCH_MAX_CHARS = 40_000;
75
- /** The user's turn is shown up to this many characters, the assistant's turn up to the second number. */
76
- exports.DIGEST_USER_TURN_MAX_CHARS = 4_000;
77
- exports.DIGEST_ASSISTANT_TURN_MAX_CHARS = 300;
78
- /** A note is shown at most this long (its head, then its tail, when longer). */
79
- exports.DIGEST_NOTE_MAX_CHARS = 20_000;
80
- exports.DIGEST_MAX_OUTPUT_TOKENS = 8_000;
81
- exports.DIGEST_TIMEOUT_MS = 180_000;
82
- exports.DIGEST_RETRIES_DEFAULT = 3;
83
- const DIGEST_ATTEMPTS = 2;
84
- const DIGEST_MAX_ITEMS_PER_NOTE = 40;
85
- /** What the rendered table may weigh at search time, in gpt-tokenizer tokens (table header and footer included). */
86
- exports.DIGEST_DEFAULT_TOKENS = 600;
87
- /** At most this many of the retrieval window's notes are read, like the evidence pass's 20 sessions. */
88
- exports.DIGEST_MAX_WINDOW_NOTES = 20;
89
- exports.DIGEST_HIT_PATH = '_renor/digest';
90
- function digestsEnabled(env = process.env) {
91
- const raw = (env.RENOR_DIGESTS ?? '').trim().toLowerCase();
92
- if (raw === '' || raw === '0' || raw === 'off' || raw === 'false' || raw === 'no')
93
- return false;
94
- if (raw === '1' || raw === 'on' || raw === 'true' || raw === 'yes')
95
- return true;
96
- // A typo must not quietly measure the control arm twice.
97
- throw new Error(`Invalid RENOR_DIGESTS "${raw}": must be on | off`);
98
- }
99
- exports.DIGEST_MAX_USD_PER_DAY_DEFAULT = 0.5;
100
- exports.DIGEST_PRICE_IN_DEFAULT = 0.225;
101
- exports.DIGEST_PRICE_OUT_DEFAULT = 0.75;
102
- function positiveNumber(raw, fallback, name) {
103
- const text = (raw ?? '').trim();
104
- if (text === '')
105
- return fallback;
106
- const value = Number(text);
107
- // A typo must not quietly lift or remove a cost guard.
108
- if (!Number.isFinite(value) || value <= 0)
109
- throw new Error(`Invalid ${name} "${text}": must be a positive number`);
110
- return value;
111
- }
112
- function boundedInt(raw, fallback, min, max) {
113
- const value = Number((raw ?? '').trim());
114
- if ((raw ?? '').trim() === '' || !Number.isFinite(value))
115
- return fallback;
116
- return Math.min(max, Math.max(min, Math.floor(value)));
117
- }
118
- /**
119
- * The digest model's provider: the digest slot (RENOR_DIGEST_BASE_URL + RENOR_DIGEST_API_KEY(S)) and nothing else.
120
- *
121
- * A digest call sends the text of notes (clipped, see DIGEST_NOTE_MAX_CHARS), and the Privacy Policy, the LGPD page and
122
- * the register of processing (ROPA P13) name one processor for it. So this never borrows the evidence provider's key or
123
- * URL, the session router's, or an Ollama Cloud key, and the transport's default URL (Ollama Cloud) is never reached:
124
- * with the slot unset no client is built and nothing is sent. Pointing the slot at a provider those pages do not name
125
- * means updating them first.
126
- */
127
- function resolveDigestProvider(env = process.env) {
128
- const keys = [...(0, keyPool_1.splitKeyList)(env.RENOR_DIGEST_API_KEYS), ...(0, keyPool_1.splitKeyList)(env.RENOR_DIGEST_API_KEY)];
129
- return { baseUrl: env.RENOR_DIGEST_BASE_URL?.trim() || undefined, keys };
130
- }
131
- /** Env -> options, or null when the flag is off. */
132
- function resolveDigestOptions(config, env = process.env) {
133
- if (!digestsEnabled(env))
134
- return null;
135
- const effortRaw = (env.RENOR_DIGEST_REASONING_EFFORT ?? 'low').trim().toLowerCase();
136
- if (!secondPass_1.EVIDENCE_REASONING_EFFORTS.includes(effortRaw)) {
137
- throw new Error(`Invalid RENOR_DIGEST_REASONING_EFFORT "${effortRaw}": must be ${secondPass_1.EVIDENCE_REASONING_EFFORTS.join(' | ')}`);
138
- }
139
- const cacheOnlyRaw = (env.RENOR_DIGEST_CACHE_ONLY ?? '').trim().toLowerCase();
140
- const cacheOnly = cacheOnlyRaw === '1' || cacheOnlyRaw === 'on' || cacheOnlyRaw === 'true' || cacheOnlyRaw === 'yes';
141
- const retries = boundedInt(env.RENOR_DIGEST_RETRIES, exports.DIGEST_RETRIES_DEFAULT, 0, 8);
142
- const { baseUrl, keys } = resolveDigestProvider(env);
143
- return {
144
- model: env.RENOR_DIGEST_MODEL?.trim() || env.RENOR_EVIDENCE_MODEL?.trim() || secondPass_1.EVIDENCE_MODEL_DEFAULT,
145
- effort: effortRaw,
146
- cachePath: env.RENOR_DIGEST_CACHE?.trim() || node_path_1.default.join(node_path_1.default.dirname(config.dbPath), 'digest-cache.db'),
147
- cacheOnly,
148
- tokens: boundedInt(env.RENOR_DIGEST_TOKENS, exports.DIGEST_DEFAULT_TOKENS, 100, 4_000),
149
- retries,
150
- maxUsdPerDay: positiveNumber(env.RENOR_DIGEST_MAX_USD_PER_DAY, exports.DIGEST_MAX_USD_PER_DAY_DEFAULT, 'RENOR_DIGEST_MAX_USD_PER_DAY'),
151
- priceInPerM: positiveNumber(env.RENOR_DIGEST_PRICE_IN_PER_M, exports.DIGEST_PRICE_IN_DEFAULT, 'RENOR_DIGEST_PRICE_IN_PER_M'),
152
- priceOutPerM: positiveNumber(env.RENOR_DIGEST_PRICE_OUT_PER_M, exports.DIGEST_PRICE_OUT_DEFAULT, 'RENOR_DIGEST_PRICE_OUT_PER_M'),
153
- // A client needs the slot's key AND its URL: a key alone would send notes to the transport's default endpoint.
154
- ...(keys.length > 0 && baseUrl && !cacheOnly ? { chat: (0, ollamaChat_1.createOllamaChat)({ baseUrl, keys, timeoutMs: exports.DIGEST_TIMEOUT_MS, maxRetries: retries }) } : {}),
155
- };
156
- }
157
- // ─── What the model is shown ───
158
- const TURN_HEADING = /^### Turn (\d+) — ([^\n]*?)[ \t]*$/gm;
159
- /**
160
- * The text a note is digested from. A note made of `### Turn N — Role` sections shows the user's turns whole (up to a
161
- * cap) and the assistant's turns clipped; any other note is shown whole, or its head and tail when long.
162
- */
163
- function digestInputOf(body) {
164
- const matches = [...body.matchAll(TURN_HEADING)];
165
- if (matches.length === 0)
166
- return clipLong(body);
167
- const parts = [];
168
- matches.forEach((match, index) => {
169
- const start = (match.index ?? 0) + match[0].length;
170
- const end = index + 1 < matches.length ? (matches[index + 1].index ?? body.length) : body.length;
171
- const text = body.slice(start, end).trim();
172
- const role = match[2].trim();
173
- const assistant = /^assistant$/i.test(role);
174
- const cap = assistant ? exports.DIGEST_ASSISTANT_TURN_MAX_CHARS : exports.DIGEST_USER_TURN_MAX_CHARS;
175
- const shown = text.length > cap ? `${text.slice(0, cap)} …` : text;
176
- parts.push(`[Turn ${match[1]} ${role}]\n${shown}`);
177
- });
178
- return clipLong(parts.join('\n\n'));
179
- }
180
- function clipLong(text) {
181
- if (text.length <= exports.DIGEST_NOTE_MAX_CHARS)
182
- return text;
183
- return `${text.slice(0, 14_000)}\n\n[... middle omitted ...]\n\n${text.slice(-6_000)}`;
184
- }
185
- function renderDigestPrompt(batch) {
186
- const conversations = batch.map(item => `=== CONVERSATION id=${item.id} date=${item.date} ===\n${item.text}`).join('\n\n');
187
- return exports.DIGEST_PROMPT_TEMPLATE.replace('{n}', () => String(batch.length)).replace('{conversations}', () => conversations);
188
- }
189
- const batchId = (id) => id.trim().toLowerCase().replace(/^c0*(\d+)$/, 'c$1');
190
- const clean = (value, max) => (typeof value === 'string' ? value : typeof value === 'number' ? String(value) : '').replace(/\s+/g, ' ').trim().slice(0, max);
191
- /**
192
- * `{"notes":[{id, items:[{item, action, amount, date, turn}]}]}` -> id (normalised) -> items. An entry whose items
193
- * list is missing is dropped (the note is asked again); an empty list is a real answer. An item without text is dropped.
194
- */
195
- function parseDigestReply(text) {
196
- const out = new Map();
197
- const trimmed = text.trim().replace(/^```(?:json)?\s*|\s*```$/g, '');
198
- let payload;
199
- try {
200
- payload = JSON.parse(trimmed);
201
- }
202
- catch {
203
- const match = trimmed.match(/\{[\s\S]*\}/);
204
- if (!match)
205
- return out;
206
- try {
207
- payload = JSON.parse(match[0]);
208
- }
209
- catch {
210
- return out;
211
- }
212
- }
213
- const notes = payload && typeof payload === 'object' ? payload.notes : undefined;
214
- if (!Array.isArray(notes))
215
- return out;
216
- for (const note of notes) {
217
- if (!note || typeof note !== 'object' || Array.isArray(note))
218
- continue;
219
- const record = note;
220
- const id = batchId(String(record.id ?? ''));
221
- if (!id || !Array.isArray(record.items))
222
- continue;
223
- const items = [];
224
- for (const raw of record.items) {
225
- if (!raw || typeof raw !== 'object' || Array.isArray(raw))
226
- continue;
227
- const r = raw;
228
- const item = clean(r.item, 200);
229
- if (!item)
230
- continue;
231
- items.push({
232
- item,
233
- action: clean(r.action, 60),
234
- amount: clean(r.amount, 60),
235
- date: clean(r.date, 40),
236
- source: clean(r.turn ?? r.source, 30),
237
- });
238
- if (items.length >= DIGEST_MAX_ITEMS_PER_NOTE)
239
- break;
240
- }
241
- out.set(id, items);
242
- }
243
- return out;
244
- }
245
- function batchesOf(items) {
246
- const batches = [];
247
- let current = [];
248
- let chars = 0;
249
- for (const item of items) {
250
- if (current.length > 0 && (chars + item.text.length > exports.DIGEST_BATCH_MAX_CHARS || current.length >= exports.DIGEST_BATCH_MAX_NOTES)) {
251
- batches.push(current);
252
- current = [];
253
- chars = 0;
254
- }
255
- current.push(item);
256
- chars += item.text.length;
257
- }
258
- if (current.length > 0)
259
- batches.push(current);
260
- return batches;
261
- }
262
- /** The text one item is embedded as: the row a reader would see, without the date. */
263
- function digestItemText(item) {
264
- return [item.item, item.action, item.amount].filter(Boolean).join(' | ');
265
- }
266
- /**
267
- * Give every note a digest, calling the model only for notes the cache does not hold, then give every digest the
268
- * vectors of its items in the provider's embedding space. Batches run one after another.
269
- */
270
- async function buildNoteDigests(args) {
271
- const started = Date.now();
272
- const { options, cache } = args;
273
- const model = options.model;
274
- const space = (0, space_1.embeddingSpace)(args.provider);
275
- const all = (0, sessionRouter_1.listRouterNotes)(args.db, args.vaultPath, args.ledgerRelPath);
276
- const notes = args.onlyPaths ? all.filter(note => args.onlyPaths.has(note.path)) : all;
277
- const stats = {
278
- model, notes: notes.length, cached: 0, digested: 0, skippedUncached: 0, failed: 0, gone: 0, calls: 0,
279
- promptTokens: 0, completionTokens: 0, items: 0, embeddedTexts: 0, ms: 0,
280
- };
281
- const keyOf = (noteHash) => digestCache_1.DigestCache.digestKey(model, options.effort, exports.DIGEST_PROMPT_TEMPLATE, exports.DIGEST_INPUT_VERSION, noteHash);
282
- const pending = [];
283
- for (const note of notes) {
284
- const key = keyOf(note.noteHash);
285
- if (cache.getDigest(key)) {
286
- stats.cached++;
287
- continue;
288
- }
289
- if (options.cacheOnly) {
290
- stats.skippedUncached++;
291
- continue;
292
- }
293
- let raw;
294
- try {
295
- raw = await promises_1.default.readFile(note.file, 'utf8');
296
- }
297
- catch (err) {
298
- if (err.code === 'ENOENT') {
299
- stats.gone++;
300
- continue;
301
- }
302
- stats.failed++;
303
- stats.error ??= `unreadable note: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
304
- continue;
305
- }
306
- if ((0, dailyEntries_1.noteContentHash)(note.file, raw) !== note.noteHash) {
307
- // Edited after it was indexed: its digest must describe what search will find, so it waits for the next index.
308
- stats.failed++;
309
- stats.error ??= 'a note changed after it was indexed; index again';
310
- continue;
311
- }
312
- const parsed = (0, parser_1.parseNote)(note.file, raw);
313
- if (!parsed.body.trim()) {
314
- cache.putDigest(key, note.noteHash, { items: [], model });
315
- stats.digested++;
316
- continue;
317
- }
318
- pending.push({ key, noteHash: note.noteHash, date: note.date ?? '', text: digestInputOf(parsed.body) });
319
- }
320
- if (pending.length > 0 && !options.chat) {
321
- stats.failed += pending.length;
322
- stats.error ??= 'no digest model configured';
323
- }
324
- else {
325
- await digestPending(pending, options, cache, stats);
326
- }
327
- await embedDigests(notes, keyOf, cache, args.provider, space, stats);
328
- stats.ms = Date.now() - started;
329
- return stats;
330
- }
331
- async function digestPending(pending, options, cache, stats) {
332
- const batches = batchesOf(pending);
333
- for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
334
- const items = batches[batchIndex].map((entry, index) => ({ ...entry, id: `c${String(index + 1).padStart(2, '0')}` }));
335
- let prompt = renderDigestPrompt(items);
336
- const left = new Map(items.map(item => [batchId(item.id), item]));
337
- let transportFailed = false;
338
- for (let attempt = 0; attempt < DIGEST_ATTEMPTS && left.size > 0; attempt++) {
339
- let content;
340
- try {
341
- const res = await options.chat({
342
- model: options.model,
343
- messages: [{ role: 'user', content: prompt }],
344
- temperature: 0,
345
- response_format: { type: 'json_object' },
346
- max_tokens: exports.DIGEST_MAX_OUTPUT_TOKENS,
347
- reasoning_effort: options.effort,
348
- });
349
- stats.calls++;
350
- stats.promptTokens += res.promptTokens ?? 0;
351
- stats.completionTokens += (0, sessionRouter_1.billedOutputTokens)(res);
352
- content = res.content;
353
- }
354
- catch (err) {
355
- stats.calls++;
356
- stats.error ??= `digest call failed: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
357
- transportFailed = true;
358
- break;
359
- }
360
- const parsed = parseDigestReply(content);
361
- for (const [id, item] of [...left]) {
362
- const found = parsed.get(id);
363
- if (!found)
364
- continue;
365
- const dated = found.map(it => (it.date ? it : { ...it, date: item.date }));
366
- cache.putDigest(item.key, item.noteHash, { items: dated, model: options.model });
367
- stats.digested++;
368
- left.delete(id);
369
- }
370
- // A different request, so a provider-side cache cannot replay the incomplete answer.
371
- prompt += '\n';
372
- }
373
- if (left.size > 0) {
374
- stats.failed += left.size;
375
- stats.error ??= 'the digest model left some notes without a digest';
376
- }
377
- if (transportFailed) {
378
- // The transport already retried: a provider that is down or out of quota fails the next batch the same way.
379
- for (const rest of batches.slice(batchIndex + 1))
380
- stats.failed += rest.length;
381
- break;
382
- }
383
- }
384
- }
385
- const EMBED_BATCH = 64;
386
- async function embedDigests(notes, keyOf, cache, provider, space, stats) {
387
- if (space === null)
388
- return;
389
- const waiting = [];
390
- for (const note of notes) {
391
- const key = keyOf(note.noteHash);
392
- const digest = cache.getDigest(key);
393
- if (!digest || digest.items.length === 0)
394
- continue;
395
- stats.items += digest.items.length;
396
- if (cache.getVectors(key, space, digest.items.length))
397
- continue;
398
- waiting.push({ key, texts: digest.items.map(digestItemText) });
399
- }
400
- for (let i = 0; i < waiting.length; i += EMBED_BATCH) {
401
- const group = waiting.slice(i, i + EMBED_BATCH);
402
- const texts = group.flatMap(entry => entry.texts);
403
- let vectors;
404
- try {
405
- vectors = await provider.embed(texts);
406
- }
407
- catch (err) {
408
- stats.failed += group.length;
409
- stats.error ??= `digest embedding failed: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200);
410
- return;
411
- }
412
- stats.embeddedTexts += texts.length;
413
- let at = 0;
414
- for (const entry of group) {
415
- cache.putVectors(entry.key, space, vectors.slice(at, at + entry.texts.length));
416
- at += entry.texts.length;
417
- }
418
- }
419
- }
420
- /**
421
- * `renor index` / `renor rebuild`: build the digests. The caller opens and closes nothing. Not under the daily cost guard
422
- * (the background worker's, worker.ts): this is a command an operator runs, and the `[digests]` line it prints says what
423
- * it spent. It must not be run from an unattended job with RENOR_DIGESTS on.
424
- */
425
- async function digestIndexedNotes(args) {
426
- const cache = new digestCache_1.DigestCache(args.options.cachePath);
427
- try {
428
- return await buildNoteDigests({
429
- db: args.db,
430
- vaultPath: args.config.vaultPath,
431
- ledgerRelPath: args.config.memoryLedger,
432
- options: args.options,
433
- cache,
434
- provider: args.provider,
435
- onlyPaths: args.onlyPaths,
436
- });
437
- }
438
- finally {
439
- cache.close();
440
- }
441
- }
442
- function cosine(a, b) {
443
- let dot = 0;
444
- let na = 0;
445
- let nb = 0;
446
- const n = Math.min(a.length, b.length);
447
- for (let i = 0; i < n; i++) {
448
- dot += a[i] * b[i];
449
- na += a[i] * a[i];
450
- nb += b[i] * b[i];
451
- }
452
- return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb);
453
- }
454
- function rowTokens(item) {
455
- const action = item.action?.trim() ? ` | ${item.action.trim()}` : '';
456
- const amount = item.amount?.trim() ? ` | ${item.amount.trim()}` : '';
457
- const date = item.date?.trim() ? ` | ${item.date.trim()}` : '';
458
- return (0, o200k_1.countO200k)(`- ${item.item}${action}${amount}${date}\n`);
459
- }
460
- /**
461
- * The table of digest items for one search, and nothing else: the digests of the window's notes (first
462
- * DIGEST_MAX_WINDOW_NOTES distinct notes of `hits`, never the ledger or a synthetic hit), the best-fitting items by
463
- * cosine to the query, as many as fit `options.tokens`, ordered oldest first. Reads the cache and the query
464
- * embedding only.
465
- */
466
- async function digestTable(args) {
467
- const report = {
468
- status: 'skipped', windowNotes: 0, notesWithDigest: 0, itemsConsidered: 0, itemsRendered: 0, tokens: 0, text: '', items: [],
469
- };
470
- const paths = [];
471
- for (const hit of args.hits) {
472
- if (!hit.notePath || hit.notePath.startsWith('_renor/') || paths.includes(hit.notePath))
473
- continue;
474
- if (args.isLedgerNote?.(hit.notePath))
475
- continue;
476
- paths.push(hit.notePath);
477
- if (paths.length >= exports.DIGEST_MAX_WINDOW_NOTES)
478
- break;
479
- }
480
- report.windowNotes = paths.length;
481
- if (paths.length === 0)
482
- return { items: [], report: { ...report, reason: 'no-notes' } };
483
- const space = (0, space_1.embeddingSpace)(args.provider);
484
- if (space === null)
485
- return { items: [], report: { ...report, reason: 'no-embedding-space' } };
486
- let cache;
487
- try {
488
- cache = new digestCache_1.DigestCache(args.options.cachePath, { readonly: true });
489
- }
490
- catch (err) {
491
- // No cache file: nothing was ever digested here, which is the same as no digests.
492
- return { items: [], report: { ...report, reason: 'no-digests', error: `digest cache unavailable: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200) } };
493
- }
494
- try {
495
- const noteRow = args.db.prepare('SELECT note_hash, frontmatter_json, modified_at FROM notes WHERE path = ?');
496
- const candidates = [];
497
- let missingVectors = 0;
498
- paths.forEach((notePath, order) => {
499
- const row = noteRow.get(notePath);
500
- if (!row || !row.note_hash)
501
- return;
502
- const key = digestCache_1.DigestCache.digestKey(args.options.model, args.options.effort, exports.DIGEST_PROMPT_TEMPLATE, exports.DIGEST_INPUT_VERSION, row.note_hash);
503
- const digest = cache.getDigest(key);
504
- if (!digest)
505
- return;
506
- report.notesWithDigest++;
507
- if (digest.items.length === 0)
508
- return;
509
- const vectors = cache.getVectors(key, space, digest.items.length);
510
- if (!vectors) {
511
- missingVectors++;
512
- return;
513
- }
514
- const noteDate = (0, orchestrator_1.resolveSessionDate)(row.frontmatter_json, row.modified_at);
515
- digest.items.forEach((item, index) => {
516
- // A date the model wrote in words ("last Tuesday") does not order; the note's own date does.
517
- const own = (0, orchestrator_1.sessionDateMs)(item.date);
518
- const dateMs = own === Number.NEGATIVE_INFINITY ? (0, orchestrator_1.sessionDateMs)(noteDate) : own;
519
- candidates.push({ notePath, order, index, item, vector: vectors[index], dateMs });
520
- });
521
- });
522
- report.itemsConsidered = candidates.length;
523
- if (candidates.length === 0) {
524
- return { items: [], report: { ...report, reason: report.notesWithDigest === 0 ? 'no-digests' : missingVectors > 0 ? 'no-vectors' : 'no-digests' } };
525
- }
526
- const queryVector = args.queryVector
527
- ?? (args.provider.embedQuery
528
- ? await args.provider.embedQuery(args.query)
529
- : (await args.provider.embed([args.query]))[0]);
530
- const ranked = candidates
531
- .map(candidate => ({ candidate, score: cosine(queryVector, candidate.vector) }))
532
- .sort((a, b) => b.score - a.score || a.candidate.order - b.candidate.order || a.candidate.index - b.candidate.index);
533
- const frame = (0, o200k_1.countO200k)((0, attach_1.renderEvidenceTable)([]));
534
- let used = frame;
535
- const chosen = [];
536
- for (const { candidate } of ranked) {
537
- const cost = rowTokens(candidate.item);
538
- if (used + cost > args.options.tokens)
539
- break;
540
- used += cost;
541
- chosen.push(candidate);
542
- }
543
- chosen.sort((a, b) => (a.dateMs === b.dateMs ? 0 : a.dateMs < b.dateMs ? -1 : 1) || a.order - b.order || a.index - b.index);
544
- const items = chosen.map(c => ({ item: c.item.item, action: c.item.action, amount: c.item.amount, date: c.item.date }));
545
- report.status = items.length > 0 ? 'ok' : 'skipped';
546
- report.itemsRendered = items.length;
547
- report.text = items.length > 0 ? (0, attach_1.renderEvidenceTable)(items) : '';
548
- report.tokens = items.length > 0 ? (0, o200k_1.countO200k)(report.text) : 0;
549
- report.items = chosen.map(c => ({ notePath: c.notePath, source: c.item.source, item: c.item.item }));
550
- return { items, report };
551
- }
552
- finally {
553
- cache.close();
554
- }
555
- }
556
- /** The digest as one hit, rendered like the evidence table (the same rows the reader sees from the evidence pass). */
557
- function digestItemsToHit(items) {
558
- return {
559
- notePath: exports.DIGEST_HIT_PATH,
560
- chunkId: -1,
561
- headingPath: 'Digest',
562
- text: (0, attach_1.renderEvidenceTable)(items),
563
- score: 1,
564
- reason: 'digest',
565
- };
566
- }
567
- /** `renor search`: the hits with the digest hit placed after any synthetic hits already leading, and the report. */
568
- async function attachDigest(args) {
569
- try {
570
- const { items, report } = await digestTable(args);
571
- if (items.length === 0)
572
- return { hits: args.hits, report };
573
- let at = 0;
574
- while (at < args.hits.length && args.hits[at].notePath.startsWith('_renor/'))
575
- at++;
576
- return { hits: [...args.hits.slice(0, at), digestItemsToHit(items), ...args.hits.slice(at)], report };
577
- }
578
- catch (err) {
579
- // Optional augmentation: a fault here must never take down the base search, and it is said, not left silent.
580
- return {
581
- hits: args.hits,
582
- report: {
583
- status: 'skipped', reason: 'digest-error', windowNotes: 0, notesWithDigest: 0, itemsConsidered: 0,
584
- itemsRendered: 0, tokens: 0, text: '', items: [], error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
585
- },
586
- };
587
- }
588
- }
589
- /** The `reason` of the synthetic hits that lead a search page: the evidence table and the digest table. */
590
- exports.LEADING_SUMMARY_REASONS = ['evidence', 'digest'];
591
- /** How many hits at the head of a page are synthetic summary tables (they sit outside top_k and the ranking). */
592
- function leadingSummaryHits(hits) {
593
- let n = 0;
594
- while (n < hits.length && exports.LEADING_SUMMARY_REASONS.includes(hits[n].reason ?? ''))
595
- n++;
596
- return n;
597
- }