renor-engine 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/LICENSE +661 -0
  2. package/NOTICE +73 -0
  3. package/README.md +105 -2
  4. package/dist/BUILD.json +6 -0
  5. package/dist/api/apiAuth.js +43 -0
  6. package/dist/api/server.js +172 -0
  7. package/dist/buildIdentity.js +132 -0
  8. package/dist/cli-extractors.js +46 -0
  9. package/dist/cli.js +491 -0
  10. package/dist/config.js +65 -0
  11. package/dist/context/composer.js +320 -0
  12. package/dist/context/readHonesty.js +478 -0
  13. package/dist/context/rowStore.js +238 -0
  14. package/dist/conversation/formatter.js +81 -0
  15. package/dist/conversation/parser.js +189 -0
  16. package/dist/conversation/types.js +2 -0
  17. package/dist/conversation/writer.js +42 -0
  18. package/dist/db/client.js +147 -0
  19. package/dist/db/indexSalvage.js +81 -0
  20. package/dist/db/migrations.js +642 -0
  21. package/dist/db/readCache.js +109 -0
  22. package/dist/db/schema.js +5 -0
  23. package/dist/db/statements.js +89 -0
  24. package/dist/digests/digestCache.js +142 -0
  25. package/dist/digests/digests.js +597 -0
  26. package/dist/digests/worker.js +348 -0
  27. package/dist/embeddings/caching.js +337 -0
  28. package/dist/embeddings/coalescing.js +177 -0
  29. package/dist/embeddings/interactive.js +103 -0
  30. package/dist/embeddings/local.js +68 -0
  31. package/dist/embeddings/openai.js +105 -0
  32. package/dist/embeddings/rebuild.js +166 -0
  33. package/dist/embeddings/simdScan.js +258 -0
  34. package/dist/embeddings/space.js +21 -0
  35. package/dist/embeddings/types.js +14 -0
  36. package/dist/embeddings/vectorIndex.js +675 -0
  37. package/dist/embeddings/vertex.js +224 -0
  38. package/dist/eval/artifacts.js +42 -0
  39. package/dist/eval/cacheReport.js +139 -0
  40. package/dist/eval/comparison.js +182 -0
  41. package/dist/eval/context700-candidate.js +145 -0
  42. package/dist/eval/evalCache.js +525 -0
  43. package/dist/eval/hashEmbedding.js +35 -0
  44. package/dist/eval/longmemeval.js +56 -0
  45. package/dist/eval/metrics.js +181 -0
  46. package/dist/eval/renorbenchBaseline.js +486 -0
  47. package/dist/eval/renorbenchDocuments.js +134 -0
  48. package/dist/eval/renorbenchProduct.js +182 -0
  49. package/dist/eval/runLedger.js +279 -0
  50. package/dist/eval/runManifest.js +479 -0
  51. package/dist/eval/scoring.js +197 -0
  52. package/dist/evidence/attach.js +267 -0
  53. package/dist/evidence/background.js +147 -0
  54. package/dist/evidence/evidenceCache.js +182 -0
  55. package/dist/evidence/gate.js +37 -0
  56. package/dist/evidence/keyPool.js +85 -0
  57. package/dist/evidence/ollamaChat.js +84 -0
  58. package/dist/evidence/optin.js +165 -0
  59. package/dist/evidence/options.js +130 -0
  60. package/dist/evidence/secondPass.js +456 -0
  61. package/dist/extraction/classifyCapture.js +34 -0
  62. package/dist/extraction/decision-model.js +174 -0
  63. package/dist/extraction/eventExtractor.js +113 -0
  64. package/dist/extraction/extractionClient.js +122 -0
  65. package/dist/extraction/factExtractor.js +46 -0
  66. package/dist/extraction/factsWritePath.js +179 -0
  67. package/dist/extraction/preferenceExtractor.js +153 -0
  68. package/dist/extraction/secretGuard.js +176 -0
  69. package/dist/extraction/secretScanner.js +665 -0
  70. package/dist/extraction/typedFacts.js +224 -0
  71. package/dist/extraction/writePathAdjudication.js +110 -0
  72. package/dist/importing/apply.js +644 -0
  73. package/dist/importing/bundle-write.js +80 -0
  74. package/dist/importing/cli.js +407 -0
  75. package/dist/importing/existing.js +49 -0
  76. package/dist/importing/facts.js +123 -0
  77. package/dist/importing/forgetImported.js +79 -0
  78. package/dist/importing/hosted.js +467 -0
  79. package/dist/importing/index.js +29 -0
  80. package/dist/importing/journalLog.js +96 -0
  81. package/dist/importing/markdown.js +98 -0
  82. package/dist/importing/plan.js +206 -0
  83. package/dist/importing/secretScreen.js +113 -0
  84. package/dist/importing/serialize.js +188 -0
  85. package/dist/importing/source.js +463 -0
  86. package/dist/importing/types.js +6 -0
  87. package/dist/indexer/chunker.js +319 -0
  88. package/dist/indexer/dailyEntries.js +63 -0
  89. package/dist/indexer/hasher.js +10 -0
  90. package/dist/indexer/laneRuns.js +48 -0
  91. package/dist/indexer/o200k.js +481 -0
  92. package/dist/indexer/o200k.json +8 -0
  93. package/dist/indexer/outsideRoot.js +77 -0
  94. package/dist/indexer/parser.js +167 -0
  95. package/dist/indexer/pipeline.js +1168 -0
  96. package/dist/indexer/staleness.js +201 -0
  97. package/dist/indexer/tokenizer.js +77 -0
  98. package/dist/indexer/walker.js +184 -0
  99. package/dist/mcp/hive.js +88 -0
  100. package/dist/mcp/requestErrors.js +47 -0
  101. package/dist/mcp/server.js +396 -0
  102. package/dist/memory/agentLease.js +43 -0
  103. package/dist/memory/confinedWrite.js +315 -0
  104. package/dist/memory/controls.js +73 -0
  105. package/dist/memory/dailyNote.js +265 -0
  106. package/dist/memory/durableAppend.js +57 -0
  107. package/dist/memory/factConflicts.js +203 -0
  108. package/dist/memory/factTerms.js +321 -0
  109. package/dist/memory/hiveNameHeader.cjs +10 -0
  110. package/dist/memory/inventory.js +55 -0
  111. package/dist/memory/invisibleText.js +20 -0
  112. package/dist/memory/lifecycle.js +1756 -0
  113. package/dist/memory/limits.js +13 -0
  114. package/dist/memory/requests.js +183 -0
  115. package/dist/memory/sourceName.js +200 -0
  116. package/dist/memory/whatsNew.js +158 -0
  117. package/dist/memory/writer.js +175 -0
  118. package/dist/memory/writerConflicts.js +85 -0
  119. package/dist/memorymap/memoryMap.js +1568 -0
  120. package/dist/memorymap/replacedJournal.js +58 -0
  121. package/dist/memorymap/replacements.js +405 -0
  122. package/dist/mvp/activity.js +86 -0
  123. package/dist/mvp/agentMemory.js +1744 -0
  124. package/dist/mvp/auth.js +531 -0
  125. package/dist/mvp/cli.js +73 -0
  126. package/dist/mvp/client-store.js +462 -0
  127. package/dist/mvp/compileCache.js +249 -0
  128. package/dist/mvp/compileSeed.js +27 -0
  129. package/dist/mvp/config.js +288 -0
  130. package/dist/mvp/connectionStatus.js +80 -0
  131. package/dist/mvp/contextRelevance.js +413 -0
  132. package/dist/mvp/engine.js +1065 -0
  133. package/dist/mvp/exportAll.js +359 -0
  134. package/dist/mvp/factLanePack.js +77 -0
  135. package/dist/mvp/factMarkers.js +49 -0
  136. package/dist/mvp/factsLane.js +203 -0
  137. package/dist/mvp/hiddenSpans.js +585 -0
  138. package/dist/mvp/history.js +413 -0
  139. package/dist/mvp/hitDates.js +31 -0
  140. package/dist/mvp/hitFloor.js +36 -0
  141. package/dist/mvp/identity.js +18 -0
  142. package/dist/mvp/idle.js +113 -0
  143. package/dist/mvp/memoryDir.js +98 -0
  144. package/dist/mvp/posthogMcp.js +210 -0
  145. package/dist/mvp/readScope.js +104 -0
  146. package/dist/mvp/readUsefulness.js +222 -0
  147. package/dist/mvp/sanitize.js +48 -0
  148. package/dist/mvp/schemaRejections.js +91 -0
  149. package/dist/mvp/scope.js +127 -0
  150. package/dist/mvp/server.js +2089 -0
  151. package/dist/mvp/serverCard.js +29 -0
  152. package/dist/mvp/storedSize.js +139 -0
  153. package/dist/mvp/telemetry.js +441 -0
  154. package/dist/retrieval/adviceLines.js +195 -0
  155. package/dist/retrieval/adviceQuestion.js +44 -0
  156. package/dist/retrieval/conversationShape.js +44 -0
  157. package/dist/retrieval/dailyEntryTime.js +55 -0
  158. package/dist/retrieval/entityLookup.js +113 -0
  159. package/dist/retrieval/factsFirst.js +188 -0
  160. package/dist/retrieval/keyword.js +84 -0
  161. package/dist/retrieval/ledgerEventTime.js +51 -0
  162. package/dist/retrieval/ledgerLiveness.js +83 -0
  163. package/dist/retrieval/ledgerPresentation.js +161 -0
  164. package/dist/retrieval/noteIdentity.js +57 -0
  165. package/dist/retrieval/orchestrator.js +1086 -0
  166. package/dist/retrieval/queryHelpers.js +142 -0
  167. package/dist/retrieval/questionSpan.js +475 -0
  168. package/dist/retrieval/readerEvents.js +690 -0
  169. package/dist/retrieval/readerLayout.js +503 -0
  170. package/dist/retrieval/relativeTimeHints.js +113 -0
  171. package/dist/retrieval/semantic.js +67 -0
  172. package/dist/router/routerCache.js +106 -0
  173. package/dist/router/sessionRouter.js +726 -0
  174. package/dist/toolRefusal.js +120 -0
  175. package/dist/watcher/watcher.js +75 -0
  176. package/package.json +120 -4
@@ -0,0 +1,359 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.NOT_EXPORTED = exports.EXPORT_SECTIONS = exports.EXPORT_FORMAT_VERSION = void 0;
7
+ exports.exportEverything = exportEverything;
8
+ const promises_1 = __importDefault(require("fs/promises"));
9
+ const path_1 = __importDefault(require("path"));
10
+ const lifecycle_1 = require("../memory/lifecycle");
11
+ const dailyNote_1 = require("../memory/dailyNote");
12
+ const memoryMap_1 = require("../memorymap/memoryMap");
13
+ const parser_1 = require("../indexer/parser");
14
+ const replacements_1 = require("../memorymap/replacements");
15
+ // ---------------------------------------------------------------------------
16
+ // export_memory: everything one workspace holds, in one documented Markdown format ("renor-export/v1").
17
+ //
18
+ // The reply is two text blocks: a JSON manifest, then the Markdown document. The document is made of sections in
19
+ // a fixed order:
20
+ // ledger the fact ledger, byte for byte (what export_memory returned before this format)
21
+ // daily-entries every entry of every daily log (remember, remember_note), one item each
22
+ // notes every other live Markdown note (imports, hand-written notes), one item each, full text
23
+ // retired everything retired and not restored: notes, entries, facts, with when it was retired
24
+ // Nothing is summarised: an item carries the stored text and every piece of metadata the engine keeps for it.
25
+ // A reply is bounded (RENOR_EXPORT_MAX_BYTES); what does not fit is deferred and named in the manifest
26
+ // (`nextSection`, `nextItem`), never dropped. A reply for one section reads only the files that section needs.
27
+ // What the export leaves out is named in the manifest too (`notExported`).
28
+ // ---------------------------------------------------------------------------
29
+ exports.EXPORT_FORMAT_VERSION = 'renor-export/v1';
30
+ exports.EXPORT_SECTIONS = ['ledger', 'daily-entries', 'notes', 'retired'];
31
+ /**
32
+ * What the export leaves out, named in every manifest (`notExported`) so a reader can see the gap instead of trusting
33
+ * "everything". Keep this in step with the sections above and with docs/architecture/EXPORT-FORMAT.md.
34
+ */
35
+ exports.NOT_EXPORTED = [
36
+ { kind: 'agentRequests', what: 'ask_agent questions and answers (the agent_requests table)', reason: 'no section for them yet' },
37
+ { kind: 'importReceipts', what: 'the receipt of each applied import (.renor-imports/<batchId>.json)', reason: 'undo_import reads them; no section for them yet' },
38
+ { kind: 'activity', what: 'the write activity behind memory_map events and whats_new (the write_events table)', reason: 'no section for it yet' },
39
+ { kind: 'agentState', what: 'connected agent names, read cursors and live leases (known_agents, agent_cursors, agent_leases)', reason: 'connection state, not memory' },
40
+ { kind: 'searchHistory', what: 'the promoted history directory that search_history reads (MVP_HISTORY_DIR)', reason: 'configured outside the memory folder' },
41
+ { kind: 'searchIndex', what: 'the search index and embeddings', reason: 'derived: index_memory rebuilds them from the exported files' },
42
+ { kind: 'otherFiles', what: 'files in the memory folder that are not Markdown (.md), dot-folders other than the retired area, and symbolic links', reason: 'the export carries Markdown notes only' },
43
+ ];
44
+ /** Characters per reply: well under the MCP result limits this codebase works to. */
45
+ const DEFAULT_MAX_CHARS = 600_000;
46
+ function maxChars() {
47
+ const raw = Number(process.env.RENOR_EXPORT_MAX_BYTES);
48
+ return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_MAX_CHARS;
49
+ }
50
+ /**
51
+ * A line separator a reader may honor as a line break: \n, \r, U+2028, U+2029 and U+0085 (NEXT LINE), and the C0 controls
52
+ * that Python's str.splitlines() also breaks on: VT, FF, FS, GS and RS (U+000B, U+000C, U+001C to U+001E).
53
+ */
54
+ const LINE_SEPARATOR = /[\n\r\u000b\u000c\u001c\u001d\u001e\u2028\u2029\u0085]/;
55
+ /**
56
+ * The start of a line the exporter writes as a section or item header (`# renor-export/v1 ·`, `## note:`, `## entry:`,
57
+ * `## retired:`, `## fact:`). A line of user text inside a fence that starts this way is escaped with one more backslash
58
+ * (`\## note:`), so no line of the document that a naive reader takes as a header is in fact the user's text. The escape is
59
+ * one-to-one: the writer adds a backslash to every line that matches, backslashes included, so a reader removes exactly one
60
+ * from each line matching the same pattern with a backslash in front (see docs/architecture/EXPORT-FORMAT.md).
61
+ */
62
+ const HEADER_START = '(?=# renor-export\\/v1 \u00b7|## (?:note:|entry:|retired:|fact:))';
63
+ const HEADER_LIKE = new RegExp(`(^|[\\n\\r\\u000b\\u000c\\u001c\\u001d\\u001e\\u2028\\u2029\\u0085])(\\\\*)${HEADER_START}`, 'g');
64
+ /** The text inside a fence as the export writes it: header-like lines escaped; nothing else changes. */
65
+ function escapeFenceText(text) {
66
+ return text.replace(HEADER_LIKE, (_match, sep, slashes) => `${sep}\\${slashes}`);
67
+ }
68
+ /** A body in a fence one backtick longer than any run in it, so no line of the body closes the fence. */
69
+ function fenced(text) {
70
+ const body = escapeFenceText(text);
71
+ const longest = Math.max(0, ...[...body.matchAll(/`+/g)].map((m) => m[0].length));
72
+ const fence = '`'.repeat(Math.max(3, longest + 1));
73
+ return `${fence}\n${body}${body.endsWith('\n') ? '' : '\n'}${fence}\n`;
74
+ }
75
+ const escapedSeparator = (ch) => `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`;
76
+ /**
77
+ * A string that must stay on one line of the export document: JSON-quoted, with every line separator
78
+ * escaped to \uXXXX. JSON.stringify already escapes \n and \r but leaves U+2028, U+2029 and U+0085 raw,
79
+ * and a raw separator in a key or value starts a line the exporter never wrote (a forged `## note:`
80
+ * header, a new field). A JSON reader decodes the escape back to the stored character.
81
+ */
82
+ function oneLine(s) {
83
+ return JSON.stringify(s).replace(/[\u2028\u2029\u0085]/g, escapedSeparator);
84
+ }
85
+ /**
86
+ * A token interpolated into a document line — an item header id or a note path (`## note:${rel}`) — held
87
+ * to one line: every line separator becomes \uXXXX in place, so no stored path can start a line the
88
+ * exporter never wrote (a forged `## note:` header or section title). A normal path is unchanged.
89
+ */
90
+ function oneLineToken(s) {
91
+ return s.replace(/[\n\r\u000b\u000c\u001c\u001d\u001e\u2028\u2029\u0085]/g, escapedSeparator);
92
+ }
93
+ /**
94
+ * One metadata line: a single-line string as written, anything else as JSON, so no value changes the
95
+ * layout. The KEY is held to the same rule: a YAML double-quoted frontmatter key may carry an escaped
96
+ * newline (gray-matter decodes it to a real one), and an unchecked key would forge a line of the document.
97
+ */
98
+ function field(key, value) {
99
+ const safeKey = LINE_SEPARATOR.test(key) ? oneLine(key) : key;
100
+ const plain = typeof value === 'string' && !LINE_SEPARATOR.test(value);
101
+ return `- ${safeKey}: ${plain ? value : oneLine(value)}\n`;
102
+ }
103
+ /**
104
+ * `- finalNewline: absent` for a note whose bytes do not end in a newline. fenced() closes the body on a line of its own,
105
+ * so a reader who takes the body between the fences gets one newline more than the note holds; this line says the last
106
+ * one is not part of it. A daily entry or a retired entry is a statement, not file bytes, so it never carries one.
107
+ */
108
+ function fileEnding(text) {
109
+ return text.endsWith('\n') ? '' : field('finalNewline', 'absent');
110
+ }
111
+ function scopeOf(rel) {
112
+ return /^Memory\/Scopes\/([^/]+)\//.exec(rel)?.[1] ?? 'global';
113
+ }
114
+ /**
115
+ * Every key the exporter writes on a note or a retired note, before or after its front matter: its header, its
116
+ * replacement lines and its finalNewline marker. A front-matter key with one of these names would print a second line of
117
+ * that name, and a reader would take the forged one for the export's (`- finalNewline: absent` strips a real final
118
+ * newline). Keys only a daily entry or a fact carries (`source`, `at`, `subject`, ...) are not here: no front matter reaches
119
+ * those items, so an ordinary `source:` key stays as it is. Add a key here when a note item gets a new line.
120
+ */
121
+ const EXPORTER_KEYS = new Set([
122
+ 'kind', 'id', 'path', 'scope', 'title', 'modifiedAt', 'storedAt', 'savedAt', 'retiredAt',
123
+ 'replacedBy', 'replacedAt', 'replaces', 'finalNewline',
124
+ ]);
125
+ /** The name a colliding front-matter key is written under; a key that already starts with it is escaped too, so the map stays one-to-one. */
126
+ const FRONTMATTER_PREFIX = 'frontmatter.';
127
+ /** A front-matter key as the export writes it: unchanged, unless it would collide with an exporter key or the prefix. */
128
+ function frontmatterKey(key) {
129
+ return EXPORTER_KEYS.has(key) || key.startsWith(FRONTMATTER_PREFIX) ? `${FRONTMATTER_PREFIX}${key}` : key;
130
+ }
131
+ function frontmatterFields(rel, text) {
132
+ const note = (0, parser_1.parseNote)(rel, text);
133
+ const lines = Object.entries(note.frontmatter).map(([k, v]) => field(frontmatterKey(k), v)).join('');
134
+ return { title: note.title, lines };
135
+ }
136
+ async function liveFiles(root, ledgerAbs) {
137
+ const out = [];
138
+ async function walk(dirAbs, relDir) {
139
+ const entries = await promises_1.default.readdir(dirAbs, { withFileTypes: true });
140
+ for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
141
+ if (e.name.startsWith('.') || e.isSymbolicLink())
142
+ continue;
143
+ const rel = relDir ? `${relDir}/${e.name}` : e.name;
144
+ if (e.isDirectory())
145
+ await walk(path_1.default.join(dirAbs, e.name), rel);
146
+ else if (e.isFile() && e.name.endsWith('.md') && path_1.default.resolve(root, rel) !== ledgerAbs)
147
+ out.push(rel);
148
+ }
149
+ }
150
+ await walk(root, '');
151
+ return out.sort();
152
+ }
153
+ /**
154
+ * The lines an item carries for a replacement (memorymap/replacements.ts): `replacedBy`, `replacedAt` and, when recorded,
155
+ * `replacedSource` (and `replacedNote` when it differs) on a memory a live correction replaced, `replaces` on a correction, as
156
+ * the ids of the memories it replaces. Nothing for a memory no replacement touches, so a workspace without one exports exactly
157
+ * what it did. Empty when no replacement state was loaded, which happens only when no section that carries these lines is built.
158
+ */
159
+ function replacementFields(rep, id) {
160
+ if (!rep)
161
+ return '';
162
+ const replaced = rep.index.replacedBy(id);
163
+ // Every link a live correction made, also one a newer correction took over: the journal keeps it, so the export does.
164
+ const replaces = [...new Set(rep.records.filter(r => r.by === id).map(r => r.old))];
165
+ const by = replaced
166
+ ? field('replacedBy', replaced.by) + field('replacedAt', replaced.at)
167
+ + (replaced.source ? field('replacedSource', replaced.source) : '')
168
+ + (replaced.note ? field('replacedNote', replaced.note) : '')
169
+ : '';
170
+ return `${by}${replaces.length > 0 ? field('replaces', replaces) : ''}`;
171
+ }
172
+ /**
173
+ * Builds the sections named in `wanted` and no others: a notes page reads note files, a daily-entries page reads daily
174
+ * files, and only the ledger section reads the ledger. The replacement state is loaded only when an item needs its lines.
175
+ */
176
+ async function buildSections(ctx, wanted) {
177
+ const built = { items: {} };
178
+ const root = path_1.default.resolve(ctx.memoryDir);
179
+ const ledgerAbs = (0, lifecycle_1.resolveLedgerPath)(ctx.memoryDir, ctx.ledgerRelPath);
180
+ if (wanted.includes('ledger')) {
181
+ let ledger;
182
+ try {
183
+ ledger = await promises_1.default.readFile(ledgerAbs, 'utf-8');
184
+ }
185
+ catch (err) {
186
+ // Only a genuinely absent ledger (pristine store) is an honest empty export; any other failure is an error.
187
+ if (err.code !== 'ENOENT')
188
+ throw err;
189
+ ledger = '';
190
+ }
191
+ let ledgerEvents;
192
+ try {
193
+ ledgerEvents = (0, lifecycle_1.parseLedgerEvents)(ledger).length;
194
+ }
195
+ catch {
196
+ ledgerEvents = ledger.split('\n').filter((l) => l.startsWith('## ')).length;
197
+ }
198
+ built.ledger = ledger;
199
+ built.ledgerEvents = ledgerEvents;
200
+ // The ledger is split by line so it pages like the rest; joined, it is the file exactly.
201
+ built.items.ledger = ledger === '' ? [] : ledger.split(/(?<=\n)/);
202
+ }
203
+ const needsReplacements = wanted.some((s) => s === 'daily-entries' || s === 'notes' || s === 'retired');
204
+ const rep = needsReplacements
205
+ ? { index: await (0, replacements_1.replacementIndex)(ctx), records: await (0, replacements_1.allReplacementRecords)(ctx) }
206
+ : undefined;
207
+ if (wanted.includes('daily-entries') || wanted.includes('notes')) {
208
+ const daily = [];
209
+ const notes = [];
210
+ for (const rel of await liveFiles(root, ledgerAbs)) {
211
+ const isDaily = dailyNote_1.DAILY_FILE_RE.test(rel);
212
+ if (!wanted.includes(isDaily ? 'daily-entries' : 'notes'))
213
+ continue;
214
+ const abs = path_1.default.join(root, ...rel.split('/'));
215
+ const text = await promises_1.default.readFile(abs, 'utf-8');
216
+ if (isDaily) {
217
+ for (const e of (0, memoryMap_1.dailyEntriesForExport)(rel, text)) {
218
+ daily.push(`## ${oneLineToken(e.id)}\n\n${field('kind', 'entry')}${field('path', rel)}${field('scope', scopeOf(rel))}${field('at', e.at)}${field('source', e.source)}${replacementFields(rep, e.id)}\n${fenced(e.text)}`);
219
+ }
220
+ continue;
221
+ }
222
+ const meta = frontmatterFields(rel, text);
223
+ const modifiedAt = (await promises_1.default.stat(abs)).mtime.toISOString();
224
+ notes.push(`## note:${oneLineToken(rel)}\n\n${field('kind', 'note')}${field('path', rel)}${field('scope', scopeOf(rel))}${field('title', meta.title)}${field('modifiedAt', modifiedAt)}${meta.lines}${replacementFields(rep, `note:${rel}`)}${fileEnding(text)}\n${fenced(text)}`);
225
+ }
226
+ if (wanted.includes('daily-entries'))
227
+ built.items['daily-entries'] = daily;
228
+ if (wanted.includes('notes'))
229
+ built.items.notes = notes;
230
+ }
231
+ if (wanted.includes('retired')) {
232
+ const retired = [];
233
+ for (const r of await (0, memoryMap_1.retiredForExport)(ctx)) {
234
+ // A retired correction keeps the link it made, so a restore from this document knows what it replaces.
235
+ const made = (rep?.records ?? []).filter(rec => rec.by === r.memoryId).map(rec => rec.old);
236
+ const meta = r.kind === 'note' ? frontmatterFields(r.path, r.content) : { title: r.title, lines: '' };
237
+ retired.push({
238
+ at: r.retiredAt,
239
+ text: `## ${oneLineToken(r.rid)}\n\n${field('kind', r.kind)}${field('id', r.memoryId)}${field('path', r.path)}${field('scope', scopeOf(r.path))}${field('title', r.title)}${field('retiredAt', r.retiredAt)}`
240
+ + `${r.storedAt ? field('storedAt', r.storedAt) : ''}${r.savedAt ? field('savedAt', r.savedAt) : ''}${made.length > 0 ? field('replaces', made) : ''}${meta.lines}${r.kind === 'note' ? fileEnding(r.content) : ''}\n${fenced(r.content)}`,
241
+ });
242
+ }
243
+ for (const f of (0, lifecycle_1.listRetiredFacts)(ctx.db)) {
244
+ retired.push({
245
+ at: f.retiredAt,
246
+ text: `## ${oneLineToken(f.id)}\n\n${field('kind', 'fact')}${field('subject', f.subject)}${field('predicate', f.predicate)}${field('object', f.object)}${field('scope', f.scope)}`
247
+ + `${field('validFrom', f.validFrom)}${field('retiredAt', f.retiredAt)}${field('retiredBy', f.retiredBy)}\n`,
248
+ });
249
+ }
250
+ retired.sort((a, b) => b.at.localeCompare(a.at));
251
+ built.items.retired = retired.map((r) => r.text);
252
+ }
253
+ return built;
254
+ }
255
+ function sectionHeader(name, count, from) {
256
+ return `\n# ${exports.EXPORT_FORMAT_VERSION} · ${name} (${count} item${count === 1 ? '' : 's'}${from > 0 ? `, continuing from item ${from}` : ''})\n\n`;
257
+ }
258
+ async function exportEverything(ctx, opts) {
259
+ const budget = maxChars();
260
+ // A whole export builds every section. A reply for one section builds that section only: the others are not read, so
261
+ // their counts are null here (a whole export, with no `section`, counts them all).
262
+ const wanted = opts.section === undefined ? exports.EXPORT_SECTIONS : [opts.section];
263
+ const built = await buildSections(ctx, wanted);
264
+ const count = (name) => built.items[name]?.length ?? null;
265
+ const counts = {
266
+ ledgerEvents: built.ledgerEvents ?? null,
267
+ dailyEntries: count('daily-entries'),
268
+ notes: count('notes'),
269
+ retired: count('retired'),
270
+ };
271
+ const sections = exports.EXPORT_SECTIONS.map((name) => ({
272
+ name,
273
+ items: count(name),
274
+ ...(name === 'ledger' ? { events: built.ledgerEvents ?? null } : {}),
275
+ }));
276
+ const nextAfter = (name) => exports.EXPORT_SECTIONS[exports.EXPORT_SECTIONS.indexOf(name) + 1] ?? null;
277
+ const base = {
278
+ file: ctx.ledgerRelPath,
279
+ format: 'markdown',
280
+ formatVersion: exports.EXPORT_FORMAT_VERSION,
281
+ bytes: built.ledger?.length ?? null,
282
+ counts,
283
+ sections,
284
+ notExported: exports.NOT_EXPORTED,
285
+ };
286
+ let document = '';
287
+ const included = [];
288
+ let nextSection = null;
289
+ let nextItem;
290
+ let section;
291
+ /** Emit items of one section from `from` while they fit; at least one, so a continuation always advances. */
292
+ const emit = (name, from) => {
293
+ const items = built.items[name] ?? [];
294
+ const joiner = name === 'ledger' ? '' : '\n';
295
+ let i = from;
296
+ let chunk = name === 'ledger' && from === 0 ? '' : sectionHeader(name, items.length, from);
297
+ while (i < items.length) {
298
+ const piece = items[i] + (name === 'ledger' ? '' : joiner);
299
+ if (i > from && document.length + chunk.length + piece.length > budget)
300
+ break;
301
+ chunk += piece;
302
+ i++;
303
+ }
304
+ document += chunk;
305
+ return i;
306
+ };
307
+ if (opts.section === undefined) {
308
+ // Whole sections while they fit. The ledger starts the reply and is paged by line only if it alone is too big.
309
+ for (const name of exports.EXPORT_SECTIONS) {
310
+ const items = built.items[name] ?? [];
311
+ const size = items.reduce((n, s) => n + s.length + 1, 0) + sectionHeader(name, items.length, 0).length;
312
+ if (name === 'ledger' ? size <= budget : document.length + size <= budget) {
313
+ if (name !== 'ledger' && items.length > 0)
314
+ document += sectionHeader(name, items.length, 0) + items.join('\n');
315
+ else if (name === 'ledger')
316
+ document += built.ledger ?? '';
317
+ if (items.length > 0 || name === 'ledger')
318
+ included.push(name);
319
+ continue;
320
+ }
321
+ if (name === 'ledger') {
322
+ const end = emit('ledger', 0);
323
+ included.push('ledger');
324
+ nextSection = 'ledger';
325
+ nextItem = end;
326
+ }
327
+ else {
328
+ nextSection = name;
329
+ }
330
+ break;
331
+ }
332
+ }
333
+ else {
334
+ section = opts.section;
335
+ const from = opts.offset ?? 0;
336
+ const total = built.items[section]?.length ?? 0;
337
+ if (from > total)
338
+ throw new Error(`offset ${from} is past the end of ${section} (${total} item${total === 1 ? '' : 's'})`);
339
+ const end = emit(section, from);
340
+ included.push(section);
341
+ if (end < total) {
342
+ nextSection = section;
343
+ nextItem = end;
344
+ }
345
+ else
346
+ nextSection = nextAfter(section);
347
+ }
348
+ const truncated = section === undefined ? nextSection !== null : nextItem !== undefined;
349
+ const manifest = {
350
+ ...base,
351
+ ...(section ? { section, offset: opts.offset ?? 0 } : {}),
352
+ included,
353
+ truncated,
354
+ nextSection,
355
+ ...(nextItem !== undefined ? { nextItem } : {}),
356
+ documentChars: document.length,
357
+ };
358
+ return { manifest, document };
359
+ }
@@ -0,0 +1,77 @@
1
+ "use strict";
2
+ /**
3
+ * The fact lane's vectors laid out for one pass over them. Every call to `factLane` takes the dot product of the query with the
4
+ * unit vector of each current fact; read from the per-provider store that is one lookup and one separate array per fact, packed
5
+ * the vectors sit in a single row-major matrix in the order the lane compares them. A pack is a copy, so it is only as current as
6
+ * the snapshot and the vectors it was made from: contextRelevance.ts says when it is made again.
7
+ */
8
+ Object.defineProperty(exports, "__esModule", { value: true });
9
+ exports.packLane = packLane;
10
+ exports.bestPacked = bestPacked;
11
+ /** Copy the stored unit vectors in snapshot order. No second normalize: the store already holds unit vectors. */
12
+ function packLane(snapshot, vectors, epoch) {
13
+ const ids = [];
14
+ const vecs = [];
15
+ const missing = [];
16
+ for (const [rendering, id] of snapshot.pairs) {
17
+ const v = vectors.get(rendering);
18
+ if (!v)
19
+ missing.push(rendering);
20
+ else {
21
+ vecs.push(v);
22
+ ids.push(id);
23
+ }
24
+ }
25
+ const dim = vecs.length > 0 ? vecs[0].length : 0;
26
+ const uniform = dim > 0 && vecs.every(v => v.length === dim);
27
+ let matrix = null;
28
+ if (uniform) {
29
+ matrix = new Float32Array(vecs.length * dim);
30
+ for (let i = 0; i < vecs.length; i++)
31
+ matrix.set(vecs[i], i * dim);
32
+ }
33
+ return { snapshot, epoch, dim, ids, missing, matrix };
34
+ }
35
+ /**
36
+ * Best dot of `q` against packed rows. Four rows at a time, but each row is still summed from dimension
37
+ * 0 upward, so its cosine bits match `dot`. The first row wins a tie, including a NaN, as `dot`'s loop did.
38
+ */
39
+ function bestPacked(q, matrix, n, dim) {
40
+ if (n === 0)
41
+ return null;
42
+ let bestI = -1;
43
+ let best = 0;
44
+ const take = (score, index) => {
45
+ if (bestI < 0 || score > best) {
46
+ best = score;
47
+ bestI = index;
48
+ }
49
+ };
50
+ let i = 0;
51
+ for (; i + 3 < n; i += 4) {
52
+ let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
53
+ const o0 = i * dim;
54
+ const o1 = o0 + dim;
55
+ const o2 = o1 + dim;
56
+ const o3 = o2 + dim;
57
+ for (let d = 0; d < dim; d++) {
58
+ const qd = q[d];
59
+ s0 += qd * matrix[o0 + d];
60
+ s1 += qd * matrix[o1 + d];
61
+ s2 += qd * matrix[o2 + d];
62
+ s3 += qd * matrix[o3 + d];
63
+ }
64
+ take(s0, i);
65
+ take(s1, i + 1);
66
+ take(s2, i + 2);
67
+ take(s3, i + 3);
68
+ }
69
+ for (; i < n; i++) {
70
+ let s = 0;
71
+ const o = i * dim;
72
+ for (let d = 0; d < dim; d++)
73
+ s += q[d] * matrix[o + d];
74
+ take(s, i);
75
+ }
76
+ return { index: bestI, score: best };
77
+ }
@@ -0,0 +1,49 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.factAuthorsByEvent = factAuthorsByEvent;
4
+ exports.eventMarker = eventMarker;
5
+ exports.markFactHits = markFactHits;
6
+ const statements_1 = require("../db/statements");
7
+ const sourceName_1 = require("../memory/sourceName");
8
+ const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
9
+ /**
10
+ * The author of each saved fact among these ledger events, by event id. Ids are lower-case in and out (ledgerEventIds);
11
+ * an event with no fact is absent from the map, so a memory that is not a fact is never marked.
12
+ */
13
+ function factAuthorsByEvent(db, eventIds) {
14
+ const ids = [...new Set(eventIds)];
15
+ if (ids.length === 0)
16
+ return new Map();
17
+ const rows = (0, statements_1.listStatement)(db, ids.length, marks => `SELECT event_id AS eventId, author FROM facts WHERE event_id IN (${marks})`).all(...(0, statements_1.paddedList)(ids.map(id => id.toUpperCase())));
18
+ return new Map(rows.map(row => [row.eventId.toLowerCase(), row.author]));
19
+ }
20
+ /**
21
+ * The marker of a memory that is one or more ledger events. A chunk normally holds one event; when it holds several
22
+ * facts of several authors, none is named, and an import among them is still marked untrusted.
23
+ */
24
+ function eventMarker(eventIds, authors, reader) {
25
+ const saved = eventIds.filter(id => authors.has(id)).map(id => authors.get(id) ?? '');
26
+ if (saved.length === 0)
27
+ return {};
28
+ const named = [...new Set(saved)];
29
+ if (named.length === 1)
30
+ return (0, sourceName_1.authorMarker)(named[0], reader);
31
+ return saved.some(sourceName_1.isImportSource) ? { untrusted: true } : {};
32
+ }
33
+ /**
34
+ * Marks, in place, each search hit that is a saved fact: a ledger event by its author, and a fact an identity question
35
+ * read by the author the retrieval carried (`author`, taken off so the reply holds the marker and not the raw name).
36
+ * A hit that is none of those is left as it was, so the owner's own results are unchanged.
37
+ */
38
+ function markFactHits(db, hits, reader, isLedger) {
39
+ const authors = factAuthorsByEvent(db, hits.filter(hit => isLedger(hit.notePath ?? '')).flatMap(hit => (0, ledgerLiveness_1.ledgerEventIds)(hit)));
40
+ for (const hit of hits) {
41
+ if ('author' in hit) {
42
+ Object.assign(hit, (0, sourceName_1.authorMarker)(hit.author, reader));
43
+ delete hit.author;
44
+ }
45
+ else if (isLedger(hit.notePath ?? '')) {
46
+ Object.assign(hit, eventMarker((0, ledgerLiveness_1.ledgerEventIds)(hit), authors, reader));
47
+ }
48
+ }
49
+ }