renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,393 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.HANDOFF_NOT_EXPORTED = exports.NOT_EXPORTED = exports.EXPORT_SECTIONS = exports.EXPORT_FORMAT_VERSION = void 0;
7
- exports.notExported = notExported;
8
- exports.exportEverything = exportEverything;
9
- const promises_1 = __importDefault(require("fs/promises"));
10
- const path_1 = __importDefault(require("path"));
11
- const lifecycle_1 = require("../memory/lifecycle");
12
- const dailyNote_1 = require("../memory/dailyNote");
13
- const memoryMap_1 = require("../memorymap/memoryMap");
14
- const parser_1 = require("../indexer/parser");
15
- const handoff_1 = require("../memory/handoff");
16
- const replacements_1 = require("../memorymap/replacements");
17
- const settings_1 = require("../learning/settings");
18
- const store_1 = require("../learning/store");
19
- // ---------------------------------------------------------------------------
20
- // export_memory: everything one workspace holds, in one documented Markdown format ("renor-export/v1").
21
- //
22
- // The reply is two text blocks: a JSON manifest, then the Markdown document. The document is made of sections in
23
- // a fixed order:
24
- // ledger the fact ledger, byte for byte (what export_memory returned before this format)
25
- // daily-entries every entry of every daily log (remember, remember_note), one item each
26
- // notes every other live Markdown note (imports, hand-written notes), one item each, full text
27
- // retired everything retired and not restored: notes, entries, facts, with when it was retired
28
- // Nothing is summarised: an item carries the stored text and every piece of metadata the engine keeps for it.
29
- // A reply is bounded (RENOR_EXPORT_MAX_BYTES); what does not fit is deferred and named in the manifest
30
- // (`nextSection`, `nextItem`), never dropped. A reply for one section reads only the files that section needs.
31
- // What the export leaves out is named in the manifest too (`notExported`).
32
- // ---------------------------------------------------------------------------
33
- exports.EXPORT_FORMAT_VERSION = 'renor-export/v1';
34
- exports.EXPORT_SECTIONS = ['ledger', 'daily-entries', 'notes', 'retired'];
35
- /**
36
- * What the export leaves out, named in every manifest (`notExported`) so a reader can see the gap instead of trusting
37
- * "everything". Keep this in step with the sections above and with docs/architecture/EXPORT-FORMAT.md.
38
- */
39
- exports.NOT_EXPORTED = [
40
- { kind: 'agentRequests', what: 'ask_agent questions and answers (the agent_requests table)', reason: 'no section for them yet' },
41
- { kind: 'importReceipts', what: 'the receipt of each applied import (.renor-imports/<batchId>.json)', reason: 'undo_import reads them; no section for them yet' },
42
- { kind: 'activity', what: 'the write activity behind memory_map events and whats_new (the write_events table)', reason: 'no section for it yet' },
43
- { kind: 'agentState', what: 'connected agent names, read cursors and live leases (known_agents, agent_cursors, agent_leases)', reason: 'connection state, not memory' },
44
- { kind: 'searchHistory', what: 'the promoted history directory that search_history reads (MVP_HISTORY_DIR)', reason: 'configured outside the memory folder' },
45
- { kind: 'searchIndex', what: 'the search index and embeddings', reason: 'derived: index_memory rebuilds them from the exported files' },
46
- { kind: 'sharingStore', what: 'the sharing choice, sender state and optional text examples in .memory-index/sharing/', reason: 'consent state and sender credentials are local controls, not portable memory; the learningLog manifest field separately exports learning.sqlite' },
47
- { kind: 'otherFiles', what: 'files in the memory folder that are not Markdown (.md), dot-folders other than the retired area, and symbolic links', reason: 'the export carries Markdown notes only' },
48
- ];
49
- /**
50
- * Named in notExported only where RENOR_HANDOFF_NOTE is on: with it off there is no handoff note and no table, and the
51
- * manifest is the one it was without it.
52
- */
53
- exports.HANDOFF_NOT_EXPORTED = {
54
- kind: 'handoffNotes',
55
- what: 'the handoff note of each repository and branch (the handoff_notes table)',
56
- reason: 'session status, not memory: it expires after 7 days and stays out of every read',
57
- };
58
- /** NOT_EXPORTED, and the handoff notes where RENOR_HANDOFF_NOTE is on. */
59
- function notExported(env = process.env) {
60
- return (0, handoff_1.handoffNoteEnabled)(env) ? [...exports.NOT_EXPORTED, exports.HANDOFF_NOT_EXPORTED] : exports.NOT_EXPORTED;
61
- }
62
- /** Characters per reply: well under the MCP result limits this codebase works to. */
63
- const DEFAULT_MAX_CHARS = 600_000;
64
- function maxChars() {
65
- const raw = Number(process.env.RENOR_EXPORT_MAX_BYTES);
66
- return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_MAX_CHARS;
67
- }
68
- /**
69
- * A line separator a reader may honor as a line break: \n, \r, U+2028, U+2029 and U+0085 (NEXT LINE), and the C0 controls
70
- * that Python's str.splitlines() also breaks on: VT, FF, FS, GS and RS (U+000B, U+000C, U+001C to U+001E).
71
- */
72
- const LINE_SEPARATOR = /[\n\r\u000b\u000c\u001c\u001d\u001e\u2028\u2029\u0085]/;
73
- /**
74
- * The start of a line the exporter writes as a section or item header (`# renor-export/v1 ·`, `## note:`, `## entry:`,
75
- * `## retired:`, `## fact:`). A line of user text inside a fence that starts this way is escaped with one more backslash
76
- * (`\## note:`), so no line of the document that a naive reader takes as a header is in fact the user's text. The escape is
77
- * one-to-one: the writer adds a backslash to every line that matches, backslashes included, so a reader removes exactly one
78
- * from each line matching the same pattern with a backslash in front (see docs/architecture/EXPORT-FORMAT.md).
79
- */
80
- const HEADER_START = '(?=# renor-export\\/v1 \u00b7|## (?:note:|entry:|retired:|fact:))';
81
- const HEADER_LIKE = new RegExp(`(^|[\\n\\r\\u000b\\u000c\\u001c\\u001d\\u001e\\u2028\\u2029\\u0085])(\\\\*)${HEADER_START}`, 'g');
82
- /** The text inside a fence as the export writes it: header-like lines escaped; nothing else changes. */
83
- function escapeFenceText(text) {
84
- return text.replace(HEADER_LIKE, (_match, sep, slashes) => `${sep}\\${slashes}`);
85
- }
86
- /** A body in a fence one backtick longer than any run in it, so no line of the body closes the fence. */
87
- function fenced(text) {
88
- const body = escapeFenceText(text);
89
- const longest = Math.max(0, ...[...body.matchAll(/`+/g)].map((m) => m[0].length));
90
- const fence = '`'.repeat(Math.max(3, longest + 1));
91
- return `${fence}\n${body}${body.endsWith('\n') ? '' : '\n'}${fence}\n`;
92
- }
93
- const escapedSeparator = (ch) => `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`;
94
- /**
95
- * A string that must stay on one line of the export document: JSON-quoted, with every line separator
96
- * escaped to \uXXXX. JSON.stringify already escapes \n and \r but leaves U+2028, U+2029 and U+0085 raw,
97
- * and a raw separator in a key or value starts a line the exporter never wrote (a forged `## note:`
98
- * header, a new field). A JSON reader decodes the escape back to the stored character.
99
- */
100
- function oneLine(s) {
101
- return JSON.stringify(s).replace(/[\u2028\u2029\u0085]/g, escapedSeparator);
102
- }
103
- /**
104
- * A token interpolated into a document line — an item header id or a note path (`## note:${rel}`) — held
105
- * to one line: every line separator becomes \uXXXX in place, so no stored path can start a line the
106
- * exporter never wrote (a forged `## note:` header or section title). A normal path is unchanged.
107
- */
108
- function oneLineToken(s) {
109
- return s.replace(/[\n\r\u000b\u000c\u001c\u001d\u001e\u2028\u2029\u0085]/g, escapedSeparator);
110
- }
111
- /**
112
- * One metadata line: a single-line string as written, anything else as JSON, so no value changes the
113
- * layout. The KEY is held to the same rule: a YAML double-quoted frontmatter key may carry an escaped
114
- * newline (gray-matter decodes it to a real one), and an unchecked key would forge a line of the document.
115
- */
116
- function field(key, value) {
117
- const safeKey = LINE_SEPARATOR.test(key) ? oneLine(key) : key;
118
- const plain = typeof value === 'string' && !LINE_SEPARATOR.test(value);
119
- return `- ${safeKey}: ${plain ? value : oneLine(value)}\n`;
120
- }
121
- /**
122
- * `- finalNewline: absent` for a note whose bytes do not end in a newline. fenced() closes the body on a line of its own,
123
- * so a reader who takes the body between the fences gets one newline more than the note holds; this line says the last
124
- * one is not part of it. A daily entry or a retired entry is a statement, not file bytes, so it never carries one.
125
- */
126
- function fileEnding(text) {
127
- return text.endsWith('\n') ? '' : field('finalNewline', 'absent');
128
- }
129
- function scopeOf(rel) {
130
- return /^Memory\/Scopes\/([^/]+)\//.exec(rel)?.[1] ?? 'global';
131
- }
132
- /**
133
- * Every key the exporter writes on a note or a retired note, before or after its front matter: its header, its
134
- * replacement lines and its finalNewline marker. A front-matter key with one of these names would print a second line of
135
- * that name, and a reader would take the forged one for the export's (`- finalNewline: absent` strips a real final
136
- * newline). Keys only a daily entry or a fact carries (`source`, `at`, `subject`, ...) are not here: no front matter reaches
137
- * those items, so an ordinary `source:` key stays as it is. Add a key here when a note item gets a new line.
138
- */
139
- const EXPORTER_KEYS = new Set([
140
- 'kind', 'id', 'path', 'scope', 'title', 'modifiedAt', 'storedAt', 'savedAt', 'retiredAt',
141
- 'replacedBy', 'replacedAt', 'replaces', 'finalNewline',
142
- ]);
143
- /** The name a colliding front-matter key is written under; a key that already starts with it is escaped too, so the map stays one-to-one. */
144
- const FRONTMATTER_PREFIX = 'frontmatter.';
145
- /** A front-matter key as the export writes it: unchanged, unless it would collide with an exporter key or the prefix. */
146
- function frontmatterKey(key) {
147
- return EXPORTER_KEYS.has(key) || key.startsWith(FRONTMATTER_PREFIX) ? `${FRONTMATTER_PREFIX}${key}` : key;
148
- }
149
- function frontmatterFields(rel, text) {
150
- const note = (0, parser_1.parseNote)(rel, text);
151
- const lines = Object.entries(note.frontmatter).map(([k, v]) => field(frontmatterKey(k), v)).join('');
152
- return { title: note.title, lines };
153
- }
154
- async function liveFiles(root, ledgerAbs) {
155
- const out = [];
156
- async function walk(dirAbs, relDir) {
157
- const entries = await promises_1.default.readdir(dirAbs, { withFileTypes: true });
158
- for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
159
- if (e.name.startsWith('.') || e.isSymbolicLink())
160
- continue;
161
- const rel = relDir ? `${relDir}/${e.name}` : e.name;
162
- if (e.isDirectory())
163
- await walk(path_1.default.join(dirAbs, e.name), rel);
164
- else if (e.isFile() && e.name.endsWith('.md') && path_1.default.resolve(root, rel) !== ledgerAbs)
165
- out.push(rel);
166
- }
167
- }
168
- await walk(root, '');
169
- return out.sort();
170
- }
171
- /**
172
- * The lines an item carries for a replacement (memorymap/replacements.ts): `replacedBy`, `replacedAt` and, when recorded,
173
- * `replacedSource` (and `replacedNote` when it differs) on a memory a live correction replaced, `replaces` on a correction, as
174
- * the ids of the memories it replaces. Nothing for a memory no replacement touches, so a workspace without one exports exactly
175
- * what it did. Empty when no replacement state was loaded, which happens only when no section that carries these lines is built.
176
- */
177
- function replacementFields(rep, id) {
178
- if (!rep)
179
- return '';
180
- const replaced = rep.index.replacedBy(id);
181
- // Every link a live correction made, also one a newer correction took over: the journal keeps it, so the export does.
182
- const replaces = [...new Set(rep.records.filter(r => r.by === id).map(r => r.old))];
183
- const by = replaced
184
- ? field('replacedBy', replaced.by) + field('replacedAt', replaced.at)
185
- + (replaced.source ? field('replacedSource', replaced.source) : '')
186
- + (replaced.note ? field('replacedNote', replaced.note) : '')
187
- : '';
188
- return `${by}${replaces.length > 0 ? field('replaces', replaces) : ''}`;
189
- }
190
- /**
191
- * Builds the sections named in `wanted` and no others: a notes page reads note files, a daily-entries page reads daily
192
- * files, and only the ledger section reads the ledger. The replacement state is loaded only when an item needs its lines.
193
- */
194
- async function buildSections(ctx, wanted) {
195
- const built = { items: {} };
196
- const root = path_1.default.resolve(ctx.memoryDir);
197
- const ledgerAbs = (0, lifecycle_1.resolveLedgerPath)(ctx.memoryDir, ctx.ledgerRelPath);
198
- if (wanted.includes('ledger')) {
199
- let ledger;
200
- try {
201
- ledger = await promises_1.default.readFile(ledgerAbs, 'utf-8');
202
- }
203
- catch (err) {
204
- // Only a genuinely absent ledger (pristine store) is an honest empty export; any other failure is an error.
205
- if (err.code !== 'ENOENT')
206
- throw err;
207
- ledger = '';
208
- }
209
- let ledgerEvents;
210
- try {
211
- ledgerEvents = (0, lifecycle_1.parseLedgerEvents)(ledger).length;
212
- }
213
- catch {
214
- ledgerEvents = ledger.split('\n').filter((l) => l.startsWith('## ')).length;
215
- }
216
- built.ledger = ledger;
217
- built.ledgerEvents = ledgerEvents;
218
- // The ledger is split by line so it pages like the rest; joined, it is the file exactly.
219
- built.items.ledger = ledger === '' ? [] : ledger.split(/(?<=\n)/);
220
- }
221
- const needsReplacements = wanted.some((s) => s === 'daily-entries' || s === 'notes' || s === 'retired');
222
- const rep = needsReplacements
223
- ? { index: await (0, replacements_1.replacementIndex)(ctx), records: await (0, replacements_1.allReplacementRecords)(ctx) }
224
- : undefined;
225
- if (wanted.includes('daily-entries') || wanted.includes('notes')) {
226
- const daily = [];
227
- const notes = [];
228
- for (const rel of await liveFiles(root, ledgerAbs)) {
229
- const isDaily = dailyNote_1.DAILY_FILE_RE.test(rel);
230
- if (!wanted.includes(isDaily ? 'daily-entries' : 'notes'))
231
- continue;
232
- const abs = path_1.default.join(root, ...rel.split('/'));
233
- const text = await promises_1.default.readFile(abs, 'utf-8');
234
- if (isDaily) {
235
- for (const e of (0, memoryMap_1.dailyEntriesForExport)(rel, text)) {
236
- daily.push(`## ${oneLineToken(e.id)}\n\n${field('kind', 'entry')}${field('path', rel)}${field('scope', scopeOf(rel))}${field('at', e.at)}${field('source', e.source)}${replacementFields(rep, e.id)}\n${fenced(e.text)}`);
237
- }
238
- continue;
239
- }
240
- const meta = frontmatterFields(rel, text);
241
- const modifiedAt = (await promises_1.default.stat(abs)).mtime.toISOString();
242
- notes.push(`## note:${oneLineToken(rel)}\n\n${field('kind', 'note')}${field('path', rel)}${field('scope', scopeOf(rel))}${field('title', meta.title)}${field('modifiedAt', modifiedAt)}${meta.lines}${replacementFields(rep, `note:${rel}`)}${fileEnding(text)}\n${fenced(text)}`);
243
- }
244
- if (wanted.includes('daily-entries'))
245
- built.items['daily-entries'] = daily;
246
- if (wanted.includes('notes'))
247
- built.items.notes = notes;
248
- }
249
- if (wanted.includes('retired')) {
250
- const retired = [];
251
- for (const r of await (0, memoryMap_1.retiredForExport)(ctx)) {
252
- // A retired correction keeps the link it made, so a restore from this document knows what it replaces.
253
- const made = (rep?.records ?? []).filter(rec => rec.by === r.memoryId).map(rec => rec.old);
254
- const meta = r.kind === 'note' ? frontmatterFields(r.path, r.content) : { title: r.title, lines: '' };
255
- retired.push({
256
- at: r.retiredAt,
257
- text: `## ${oneLineToken(r.rid)}\n\n${field('kind', r.kind)}${field('id', r.memoryId)}${field('path', r.path)}${field('scope', scopeOf(r.path))}${field('title', r.title)}${field('retiredAt', r.retiredAt)}`
258
- + `${r.storedAt ? field('storedAt', r.storedAt) : ''}${r.savedAt ? field('savedAt', r.savedAt) : ''}${made.length > 0 ? field('replaces', made) : ''}${meta.lines}${r.kind === 'note' ? fileEnding(r.content) : ''}\n${fenced(r.content)}`,
259
- });
260
- }
261
- for (const f of (0, lifecycle_1.listRetiredFacts)(ctx.db)) {
262
- retired.push({
263
- at: f.retiredAt,
264
- text: `## ${oneLineToken(f.id)}\n\n${field('kind', 'fact')}${field('subject', f.subject)}${field('predicate', f.predicate)}${field('object', f.object)}${field('scope', f.scope)}`
265
- + `${field('validFrom', f.validFrom)}${field('retiredAt', f.retiredAt)}${field('retiredBy', f.retiredBy)}\n`,
266
- });
267
- }
268
- retired.sort((a, b) => b.at.localeCompare(a.at));
269
- built.items.retired = retired.map((r) => r.text);
270
- }
271
- return built;
272
- }
273
- /** The portable contents of learning.sqlite, when that optional log exists. */
274
- function exportLearningLog(memoryDir) {
275
- const file = (0, settings_1.learningDbPath)(memoryDir);
276
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(), rawQueries: false, create: false });
277
- if (log === null)
278
- return null;
279
- try {
280
- return { file: '.memory-index/learning.sqlite', data: log.exportLog() };
281
- }
282
- finally {
283
- log.close();
284
- }
285
- }
286
- function sectionHeader(name, count, from) {
287
- return `\n# ${exports.EXPORT_FORMAT_VERSION} · ${name} (${count} item${count === 1 ? '' : 's'}${from > 0 ? `, continuing from item ${from}` : ''})\n\n`;
288
- }
289
- async function exportEverything(ctx, opts) {
290
- const budget = maxChars();
291
- // A whole export builds every section. A reply for one section builds that section only: the others are not read, so
292
- // their counts are null here (a whole export, with no `section`, counts them all).
293
- const wanted = opts.section === undefined ? exports.EXPORT_SECTIONS : [opts.section];
294
- const built = await buildSections(ctx, wanted);
295
- const count = (name) => built.items[name]?.length ?? null;
296
- const counts = {
297
- ledgerEvents: built.ledgerEvents ?? null,
298
- dailyEntries: count('daily-entries'),
299
- notes: count('notes'),
300
- retired: count('retired'),
301
- };
302
- const sections = exports.EXPORT_SECTIONS.map((name) => ({
303
- name,
304
- items: count(name),
305
- ...(name === 'ledger' ? { events: built.ledgerEvents ?? null } : {}),
306
- }));
307
- const nextAfter = (name) => exports.EXPORT_SECTIONS[exports.EXPORT_SECTIONS.indexOf(name) + 1] ?? null;
308
- const base = {
309
- file: ctx.ledgerRelPath,
310
- format: 'markdown',
311
- formatVersion: exports.EXPORT_FORMAT_VERSION,
312
- bytes: built.ledger?.length ?? null,
313
- counts,
314
- sections,
315
- notExported: notExported(),
316
- // learning.sqlite is user-owned workspace data, not a rebuildable search index. Export its
317
- // portable rows (never its private HMAC secret) alongside the Markdown memory document.
318
- learningLog: exportLearningLog(ctx.memoryDir),
319
- };
320
- let document = '';
321
- const included = [];
322
- let nextSection = null;
323
- let nextItem;
324
- let section;
325
- /** Emit items of one section from `from` while they fit; at least one, so a continuation always advances. */
326
- const emit = (name, from) => {
327
- const items = built.items[name] ?? [];
328
- const joiner = name === 'ledger' ? '' : '\n';
329
- let i = from;
330
- let chunk = name === 'ledger' && from === 0 ? '' : sectionHeader(name, items.length, from);
331
- while (i < items.length) {
332
- const piece = items[i] + (name === 'ledger' ? '' : joiner);
333
- if (i > from && document.length + chunk.length + piece.length > budget)
334
- break;
335
- chunk += piece;
336
- i++;
337
- }
338
- document += chunk;
339
- return i;
340
- };
341
- if (opts.section === undefined) {
342
- // Whole sections while they fit. The ledger starts the reply and is paged by line only if it alone is too big.
343
- for (const name of exports.EXPORT_SECTIONS) {
344
- const items = built.items[name] ?? [];
345
- const size = items.reduce((n, s) => n + s.length + 1, 0) + sectionHeader(name, items.length, 0).length;
346
- if (name === 'ledger' ? size <= budget : document.length + size <= budget) {
347
- if (name !== 'ledger' && items.length > 0)
348
- document += sectionHeader(name, items.length, 0) + items.join('\n');
349
- else if (name === 'ledger')
350
- document += built.ledger ?? '';
351
- if (items.length > 0 || name === 'ledger')
352
- included.push(name);
353
- continue;
354
- }
355
- if (name === 'ledger') {
356
- const end = emit('ledger', 0);
357
- included.push('ledger');
358
- nextSection = 'ledger';
359
- nextItem = end;
360
- }
361
- else {
362
- nextSection = name;
363
- }
364
- break;
365
- }
366
- }
367
- else {
368
- section = opts.section;
369
- const from = opts.offset ?? 0;
370
- const total = built.items[section]?.length ?? 0;
371
- if (from > total)
372
- throw new Error(`offset ${from} is past the end of ${section} (${total} item${total === 1 ? '' : 's'})`);
373
- const end = emit(section, from);
374
- included.push(section);
375
- if (end < total) {
376
- nextSection = section;
377
- nextItem = end;
378
- }
379
- else
380
- nextSection = nextAfter(section);
381
- }
382
- const truncated = section === undefined ? nextSection !== null : nextItem !== undefined;
383
- const manifest = {
384
- ...base,
385
- ...(section ? { section, offset: opts.offset ?? 0 } : {}),
386
- included,
387
- truncated,
388
- nextSection,
389
- ...(nextItem !== undefined ? { nextItem } : {}),
390
- documentChars: document.length,
391
- };
392
- return { manifest, document };
393
- }
@@ -1,77 +0,0 @@
1
- "use strict";
2
- /**
3
- * The fact lane's vectors laid out for one pass over them. Every call to `factLane` takes the dot product of the query with the
4
- * unit vector of each current fact; read from the per-provider store that is one lookup and one separate array per fact, packed
5
- * the vectors sit in a single row-major matrix in the order the lane compares them. A pack is a copy, so it is only as current as
6
- * the snapshot and the vectors it was made from: contextRelevance.ts says when it is made again.
7
- */
8
- Object.defineProperty(exports, "__esModule", { value: true });
9
- exports.packLane = packLane;
10
- exports.bestPacked = bestPacked;
11
- /** Copy the stored unit vectors in snapshot order. No second normalize: the store already holds unit vectors. */
12
- function packLane(snapshot, vectors, epoch) {
13
- const ids = [];
14
- const vecs = [];
15
- const missing = [];
16
- for (const [rendering, id] of snapshot.pairs) {
17
- const v = vectors.get(rendering);
18
- if (!v)
19
- missing.push(rendering);
20
- else {
21
- vecs.push(v);
22
- ids.push(id);
23
- }
24
- }
25
- const dim = vecs.length > 0 ? vecs[0].length : 0;
26
- const uniform = dim > 0 && vecs.every(v => v.length === dim);
27
- let matrix = null;
28
- if (uniform) {
29
- matrix = new Float32Array(vecs.length * dim);
30
- for (let i = 0; i < vecs.length; i++)
31
- matrix.set(vecs[i], i * dim);
32
- }
33
- return { snapshot, epoch, dim, ids, missing, matrix };
34
- }
35
- /**
36
- * Best dot of `q` against packed rows. Four rows at a time, but each row is still summed from dimension
37
- * 0 upward, so its cosine bits match `dot`. The first row wins a tie, including a NaN, as `dot`'s loop did.
38
- */
39
- function bestPacked(q, matrix, n, dim) {
40
- if (n === 0)
41
- return null;
42
- let bestI = -1;
43
- let best = 0;
44
- const take = (score, index) => {
45
- if (bestI < 0 || score > best) {
46
- best = score;
47
- bestI = index;
48
- }
49
- };
50
- let i = 0;
51
- for (; i + 3 < n; i += 4) {
52
- let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
53
- const o0 = i * dim;
54
- const o1 = o0 + dim;
55
- const o2 = o1 + dim;
56
- const o3 = o2 + dim;
57
- for (let d = 0; d < dim; d++) {
58
- const qd = q[d];
59
- s0 += qd * matrix[o0 + d];
60
- s1 += qd * matrix[o1 + d];
61
- s2 += qd * matrix[o2 + d];
62
- s3 += qd * matrix[o3 + d];
63
- }
64
- take(s0, i);
65
- take(s1, i + 1);
66
- take(s2, i + 2);
67
- take(s3, i + 3);
68
- }
69
- for (; i < n; i++) {
70
- let s = 0;
71
- const o = i * dim;
72
- for (let d = 0; d < dim; d++)
73
- s += q[d] * matrix[o + d];
74
- take(s, i);
75
- }
76
- return { index: bestI, score: best };
77
- }
@@ -1,65 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.factAuthorsByEvent = factAuthorsByEvent;
4
- exports.eventMarker = eventMarker;
5
- exports.markFactHits = markFactHits;
6
- exports.claimsReservedEntrySource = claimsReservedEntrySource;
7
- const statements_1 = require("../db/statements");
8
- const sourceName_1 = require("../memory/sourceName");
9
- const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
10
- const dailyNote_1 = require("../memory/dailyNote");
11
- /**
12
- * The author of each saved fact among these ledger events, by event id. Ids are lower-case in and out (ledgerEventIds);
13
- * an event with no fact is absent from the map, so a memory that is not a fact is never marked.
14
- */
15
- function factAuthorsByEvent(db, eventIds) {
16
- const ids = [...new Set(eventIds)];
17
- if (ids.length === 0)
18
- return new Map();
19
- const rows = (0, statements_1.listStatement)(db, ids.length, marks => `SELECT event_id AS eventId, author FROM facts WHERE event_id IN (${marks})`).all(...(0, statements_1.paddedList)(ids.map(id => id.toUpperCase())));
20
- return new Map(rows.map(row => [row.eventId.toLowerCase(), row.author]));
21
- }
22
- /**
23
- * The marker of a memory that is one or more ledger events. A chunk normally holds one event; when it holds several
24
- * facts of several authors, none is named, and an import or a reserved source among them is still marked untrusted.
25
- */
26
- function eventMarker(eventIds, authors, reader) {
27
- const saved = eventIds.filter(id => authors.has(id)).map(id => authors.get(id) ?? '');
28
- if (saved.length === 0)
29
- return {};
30
- const named = [...new Set(saved)];
31
- if (named.length === 1)
32
- return (0, sourceName_1.authorMarker)(named[0], reader);
33
- return saved.some(author => (0, sourceName_1.isImportSource)(author) || (0, sourceName_1.isReservedSource)(author)) ? { untrusted: true } : {};
34
- }
35
- /**
36
- * Marks, in place, each search hit that is a saved fact: a ledger event by its author, and a fact an identity question
37
- * read by the author the retrieval carried (`author`, taken off so the reply holds the marker and not the raw name).
38
- * A hit that is none of those is left as it was, so the owner's own results are unchanged.
39
- */
40
- function markFactHits(db, hits, reader, isLedger) {
41
- const authors = factAuthorsByEvent(db, hits.filter(hit => isLedger(hit.notePath ?? '')).flatMap(hit => (0, ledgerLiveness_1.ledgerEventIds)(hit)));
42
- for (const hit of hits) {
43
- if ('author' in hit) {
44
- Object.assign(hit, (0, sourceName_1.authorMarker)(hit.author, reader));
45
- delete hit.author;
46
- }
47
- else if (isLedger(hit.notePath ?? '')) {
48
- Object.assign(hit, eventMarker((0, ledgerLiveness_1.ledgerEventIds)(hit), authors, reader));
49
- }
50
- }
51
- }
52
- /**
53
- * Whether a chunk of a daily log holds an entry whose header claims a reserved source (`- [09:00:00] (system) ...`): an entry
54
- * saved before the reserved-identity rule, or written into the file by hand. The search reads (get_context, search_memory,
55
- * search_text) mark such a chunk `untrusted`, as memory_map and read_memory mark the entry. A chunk is the unit a reply
56
- * carries, so the mark is on the whole chunk; text is never changed.
57
- */
58
- function claimsReservedEntrySource(notePath, text) {
59
- if (typeof notePath !== 'string' || typeof text !== 'string' || !dailyNote_1.DAILY_FILE_RE.test(notePath))
60
- return false;
61
- return text.split('\n').some(line => {
62
- const header = (0, dailyNote_1.parseEntryHeader)(line);
63
- return header !== null && (0, sourceName_1.isReservedSource)(header.source);
64
- });
65
- }