renor-engine 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +27 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +3 -3
  4. package/dist/cli.js +1656 -490
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -548
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/preferenceExtractor.js +0 -150
  75. package/dist/extraction/secretGuard.js +0 -176
  76. package/dist/extraction/secretScanner.js +0 -665
  77. package/dist/extraction/typedFacts.js +0 -224
  78. package/dist/extraction/writePathAdjudication.js +0 -110
  79. package/dist/importing/apply.js +0 -672
  80. package/dist/importing/bundle-write.js +0 -80
  81. package/dist/importing/cli.js +0 -408
  82. package/dist/importing/existing.js +0 -51
  83. package/dist/importing/facts.js +0 -121
  84. package/dist/importing/forgetImported.js +0 -79
  85. package/dist/importing/hosted.js +0 -467
  86. package/dist/importing/journalLog.js +0 -96
  87. package/dist/importing/markdown.js +0 -98
  88. package/dist/importing/plan.js +0 -201
  89. package/dist/importing/secretScreen.js +0 -113
  90. package/dist/importing/serialize.js +0 -188
  91. package/dist/importing/source.js +0 -463
  92. package/dist/importing/types.js +0 -6
  93. package/dist/indexer/chunker.js +0 -319
  94. package/dist/indexer/dailyEntries.js +0 -63
  95. package/dist/indexer/hasher.js +0 -10
  96. package/dist/indexer/laneRuns.js +0 -48
  97. package/dist/indexer/o200k.js +0 -481
  98. package/dist/indexer/outsideRoot.js +0 -77
  99. package/dist/indexer/parser.js +0 -170
  100. package/dist/indexer/pipeline.js +0 -988
  101. package/dist/indexer/staleness.js +0 -201
  102. package/dist/indexer/tokenizer.js +0 -77
  103. package/dist/indexer/walker.js +0 -170
  104. package/dist/mcp/hive.js +0 -88
  105. package/dist/mcp/requestErrors.js +0 -47
  106. package/dist/mcp/server.js +0 -426
  107. package/dist/memory/agentLease.js +0 -43
  108. package/dist/memory/confinedWrite.js +0 -315
  109. package/dist/memory/controls.js +0 -73
  110. package/dist/memory/dailyNote.js +0 -265
  111. package/dist/memory/durableAppend.js +0 -57
  112. package/dist/memory/factConflicts.js +0 -203
  113. package/dist/memory/factTerms.js +0 -321
  114. package/dist/memory/handoff.js +0 -363
  115. package/dist/memory/hiveNameHeader.cjs +0 -10
  116. package/dist/memory/inventory.js +0 -55
  117. package/dist/memory/invisibleText.js +0 -20
  118. package/dist/memory/lifecycle.js +0 -1991
  119. package/dist/memory/limits.js +0 -13
  120. package/dist/memory/requests.js +0 -183
  121. package/dist/memory/sourceName.js +0 -276
  122. package/dist/memory/statusFacts.js +0 -61
  123. package/dist/memory/whatsNew.js +0 -158
  124. package/dist/memory/writer.js +0 -175
  125. package/dist/memory/writerConflicts.js +0 -85
  126. package/dist/memorymap/memoryMap.js +0 -1594
  127. package/dist/memorymap/replacedJournal.js +0 -58
  128. package/dist/memorymap/replacements.js +0 -410
  129. package/dist/mvp/activity.js +0 -86
  130. package/dist/mvp/agentMemory.js +0 -1800
  131. package/dist/mvp/answerCheck.js +0 -211
  132. package/dist/mvp/auth.js +0 -531
  133. package/dist/mvp/cli.js +0 -75
  134. package/dist/mvp/client-store.js +0 -462
  135. package/dist/mvp/compileCache.js +0 -242
  136. package/dist/mvp/compileSeed.js +0 -33
  137. package/dist/mvp/config.js +0 -288
  138. package/dist/mvp/connectionStatus.js +0 -80
  139. package/dist/mvp/contextDistinct.js +0 -53
  140. package/dist/mvp/contextProfile.js +0 -87
  141. package/dist/mvp/contextRelevance.js +0 -413
  142. package/dist/mvp/directorySurface.js +0 -68
  143. package/dist/mvp/engine.js +0 -1078
  144. package/dist/mvp/exportAll.js +0 -374
  145. package/dist/mvp/factLanePack.js +0 -77
  146. package/dist/mvp/factMarkers.js +0 -65
  147. package/dist/mvp/factsLane.js +0 -203
  148. package/dist/mvp/hiddenSpans.js +0 -585
  149. package/dist/mvp/history.js +0 -414
  150. package/dist/mvp/hitDates.js +0 -31
  151. package/dist/mvp/hitFloor.js +0 -36
  152. package/dist/mvp/identity.js +0 -18
  153. package/dist/mvp/idle.js +0 -113
  154. package/dist/mvp/log.js +0 -118
  155. package/dist/mvp/memoryDir.js +0 -98
  156. package/dist/mvp/posthogMcp.js +0 -210
  157. package/dist/mvp/readScope.js +0 -104
  158. package/dist/mvp/readUsefulness.js +0 -222
  159. package/dist/mvp/sanitize.js +0 -48
  160. package/dist/mvp/schemaRejections.js +0 -91
  161. package/dist/mvp/scope.js +0 -127
  162. package/dist/mvp/server.js +0 -2604
  163. package/dist/mvp/serverCard.js +0 -38
  164. package/dist/mvp/storedSize.js +0 -139
  165. package/dist/mvp/telemetry.js +0 -474
  166. package/dist/net/poolFetch.js +0 -174
  167. package/dist/recall/actionRecall.js +0 -282
  168. package/dist/recall/autoRecall.js +0 -447
  169. package/dist/recall/searchRecall.js +0 -189
  170. package/dist/retrieval/adviceLines.js +0 -195
  171. package/dist/retrieval/adviceQuestion.js +0 -44
  172. package/dist/retrieval/conversationShape.js +0 -44
  173. package/dist/retrieval/dailyEntryTime.js +0 -55
  174. package/dist/retrieval/entityLookup.js +0 -162
  175. package/dist/retrieval/factsFirst.js +0 -188
  176. package/dist/retrieval/keyword.js +0 -84
  177. package/dist/retrieval/ledgerEventTime.js +0 -51
  178. package/dist/retrieval/ledgerLiveness.js +0 -83
  179. package/dist/retrieval/ledgerPresentation.js +0 -161
  180. package/dist/retrieval/noteIdentity.js +0 -57
  181. package/dist/retrieval/orchestrator.js +0 -1171
  182. package/dist/retrieval/plainWording.js +0 -236
  183. package/dist/retrieval/queryHelpers.js +0 -142
  184. package/dist/retrieval/questionSpan.js +0 -475
  185. package/dist/retrieval/readerEvents.js +0 -690
  186. package/dist/retrieval/readerLayout.js +0 -422
  187. package/dist/retrieval/relativeTimeHints.js +0 -113
  188. package/dist/retrieval/semantic.js +0 -67
  189. package/dist/router/routerCache.js +0 -106
  190. package/dist/router/sessionRouter.js +0 -726
  191. package/dist/toolRefusal.js +0 -120
  192. package/dist/watcher/watcher.js +0 -77
  193. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,548 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.SnapshotStore = exports.INDEX_ENTRY_MODULES = exports.READ_PATH_ONLY = exports.DYNAMIC_ENV_READS = exports.READ_PATH_BINDINGS = exports.INDEX_PINNED_CONSTANTS = exports.INDEX_CODE_PATHS = exports.DEFAULT_VAULT_BASE = exports.SNAPSHOT_SCHEMA = void 0;
7
- exports.cliIndexHalf = cliIndexHalf;
8
- exports.constantDeclaration = constantDeclaration;
9
- exports.indexCodeIdentity = indexCodeIdentity;
10
- exports.indexConfig = indexConfig;
11
- exports.vaultSignature = vaultSignature;
12
- exports.vaultDirFor = vaultDirFor;
13
- exports.restoredReplay = restoredReplay;
14
- exports.snapshotReport = snapshotReport;
15
- exports.pruneSnapshots = pruneSnapshots;
16
- exports.resolveSnapshotRoot = resolveSnapshotRoot;
17
- exports.resolveVaultBase = resolveVaultBase;
18
- const node_fs_1 = __importDefault(require("node:fs"));
19
- const node_path_1 = __importDefault(require("node:path"));
20
- const node_crypto_1 = __importDefault(require("node:crypto"));
21
- const client_1 = require("../db/client");
22
- /**
23
- * Index snapshots for LongMemEval iteration runs.
24
- *
25
- * The product runner builds one memory store per question (render the haystack as markdown, run `cli index`).
26
- * That store depends only on the question's haystack and on the code and configuration that decide what the
27
- * indexer stores, so a later run that changes only retrieval, page building or reading can restore it instead
28
- * of building it again. A snapshot is the SQLite file `cli index` leaves behind, copied right after the index
29
- * and before any search touches it.
30
- *
31
- * Correctness rests on three things, all enforced here:
32
- * 1. The key is a hash of everything that reaches the stored bytes: the indexing code (INDEX_CODE_PATHS, the
33
- * index half of src/cli.ts, the lockfile), the environment the indexing code reads (scanned from those
34
- * sources, so a new flag is in the key without anyone remembering it), the embedding identity, the Node
35
- * major and platform, the vault location and the rendered haystack itself. Any difference is a miss and the
36
- * store is rebuilt.
37
- * 2. The index stores absolute paths (notes, chunks and their full-text tables), so a snapshot only works at
38
- * the path it was built at. The vault of a snapshotted question is therefore rendered at a fixed place
39
- * (vaultDirFor), independent of where the snapshots themselves live, and that place is part of the key.
40
- * Moving the snapshot directory to another machine works; changing the vault base builds a new set.
41
- * 3. A snapshot is verified (sha-256 of the file against the metadata written after it) before it is used. A
42
- * missing, truncated or altered one is a rebuild, never an error and never a silent use.
43
- *
44
- * Nothing here is used for promotion arms: they build cold (the runner refuses the combination).
45
- */
46
- exports.SNAPSHOT_SCHEMA = 'indexsnap-v1';
47
- /** Where the vault of a snapshotted question is rendered. On a RAM-backed temp directory by default (no SSD writes). */
48
- exports.DEFAULT_VAULT_BASE = '/tmp/renor-index-vault';
49
- /**
50
- * The source that decides what `cli index` stores. Directories are hashed whole. src/retrieval/orchestrator.ts
51
- * is here because the indexer imports three date helpers from it (resolveSessionDate, sessionDateMs,
52
- * boundContentDate); a change to it re-indexes, which is the safe side. Everything else retrieval, page
53
- * building, evidence and the reader is deliberately NOT here: tests/eval/index-snapshots.test.ts walks the
54
- * import graph from the index command and fails when a module appears that is neither listed here nor named in
55
- * READ_PATH_ONLY, so the boundary cannot drift unnoticed.
56
- */
57
- exports.INDEX_CODE_PATHS = [
58
- 'src/indexer',
59
- 'src/db',
60
- 'src/embeddings',
61
- 'src/extraction',
62
- 'src/memory',
63
- 'src/digests',
64
- 'src/router',
65
- 'src/net',
66
- 'src/config.ts',
67
- 'src/cli-extractors.ts',
68
- 'src/context/rowStore.ts',
69
- 'src/eval/hashEmbedding.ts',
70
- 'src/mvp/log.ts',
71
- 'src/mvp/scope.ts',
72
- 'src/toolRefusal.ts',
73
- 'src/evidence/ollamaChat.ts',
74
- 'src/evidence/keyPool.ts',
75
- 'src/evidence/options.ts',
76
- 'src/retrieval/orchestrator.ts',
77
- 'src/retrieval/entityLookup.ts',
78
- 'src/retrieval/queryHelpers.ts',
79
- ];
80
- /**
81
- * Constants the index code takes from a read-path module. The module stays out of the hash (an edit to the evidence
82
- * second pass is a search-time lever and must not re-index), but the declaration of each constant the index code
83
- * reads is hashed: the write-time digest lane and the session router fall back to EVIDENCE_MODEL_DEFAULT when no
84
- * model variable is set, so changing that default changes the digests and summaries a cold build stores. A name that
85
- * is not declared as `export const NAME ...;` in its file is an error, never a skipped entry.
86
- */
87
- exports.INDEX_PINNED_CONSTANTS = {
88
- 'src/evidence/secondPass.ts': ['EVIDENCE_MODEL_DEFAULT', 'EVIDENCE_REASONING_EFFORTS'],
89
- };
90
- /**
91
- * Every other binding that hashed code imports from a READ_PATH_ONLY module, with why it does not decide what an
92
- * index stores. tests/eval/index-snapshots.test.ts compares this with the real imports, so a new binding taken from
93
- * a read-path module fails until someone classifies it: pin it above, or list it here with the reason.
94
- * Keyed `<module>#<binding>`.
95
- */
96
- exports.READ_PATH_BINDINGS = {
97
- 'src/evidence/attach.ts#renderEvidenceTable': 'digests.ts renders a digest table into a page at search time',
98
- 'src/evidence/evidenceCache.ts#EvidenceCache': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
99
- 'src/evidence/optin.ts#EvidenceDailyBudget': 'options.ts resolveEvidenceChat, the search-time evidence pass',
100
- 'src/evidence/optin.ts#optinEndpointAllowed': 'options.ts resolveEvidenceChat, the search-time evidence pass',
101
- 'src/evidence/optin.ts#readOptinKey': 'options.ts resolveEvidenceChat, the search-time evidence pass',
102
- 'src/evidence/optin.ts#resolveEvidenceOptin': 'options.ts resolveEvidenceChat, the search-time evidence pass',
103
- 'src/evidence/optin.ts#dailyBudgetGate': 'options.ts resolveEvidenceChat, the search-time evidence pass',
104
- 'src/evidence/gate.ts#resolveEvidenceMode': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
105
- 'src/evidence/secondPass.ts#MAX_EVIDENCE_SESSIONS': 'options.ts evidence bounds, search time',
106
- 'src/evidence/secondPass.ts#EVIDENCE_SESSION_CHAR_CAP': 'options.ts evidence bounds, search time',
107
- 'src/evidence/secondPass.ts#EvidenceExtractor': 'a type, erased at build time',
108
- };
109
- /** Hashed files whose environment reads the lexical scan cannot see by name, with what covers them instead. */
110
- exports.DYNAMIC_ENV_READS = {
111
- 'src/evidence/keyPool.ts': 'reads the Ollama credential variables by a name list; credentials are never in the key',
112
- 'src/memory/statusFacts.ts': 'indexes env[STATUS_FRESHNESS_FLAG], a constant whose RENOR_ literal is in the same file and so in the scan',
113
- };
114
- /**
115
- * Modules the index command can reach through an import but does not run to produce stored state: they are
116
- * imported by orchestrator.ts / digests.ts / the router for their search-time functions. Listed so the import
117
- * walk in the test names every file it does not hash, with the reason.
118
- */
119
- exports.READ_PATH_ONLY = [
120
- 'src/evidence/attach.ts', // digests.ts imports renderEvidenceTable, used when a page is built
121
- 'src/evidence/evidenceCache.ts',
122
- 'src/evidence/gate.ts',
123
- 'src/evidence/optin.ts',
124
- 'src/evidence/secondPass.ts', // router.ts reads the EVIDENCE_MODEL_DEFAULT constant (RENOR_EVIDENCE_MODEL is in the key)
125
- 'src/mvp/contextRelevance.ts', // orchestrator: calibrated relevance bars at search time
126
- 'src/retrieval/conversationShape.ts',
127
- 'src/retrieval/dailyEntryTime.ts',
128
- 'src/retrieval/factsFirst.ts',
129
- 'src/retrieval/keyword.ts',
130
- 'src/retrieval/ledgerEventTime.ts',
131
- 'src/retrieval/ledgerLiveness.ts',
132
- 'src/retrieval/ledgerPresentation.ts',
133
- 'src/retrieval/plainWording.ts',
134
- 'src/retrieval/semantic.ts',
135
- ];
136
- /**
137
- * The modules the `index` command builds on: what src/cli.ts's setup() and index action construct and call. The import
138
- * walk in tests/eval/index-snapshots.test.ts starts here. A new dependency of the index command belongs in this list
139
- * and in INDEX_CODE_PATHS.
140
- */
141
- exports.INDEX_ENTRY_MODULES = [
142
- 'src/config.ts',
143
- 'src/db/client.ts',
144
- 'src/db/schema.ts',
145
- 'src/db/sqlite.ts',
146
- 'src/memory/lifecycle.ts',
147
- 'src/indexer/pipeline.ts',
148
- 'src/indexer/staleness.ts',
149
- 'src/embeddings/vectorIndex.ts',
150
- 'src/embeddings/openai.ts',
151
- 'src/embeddings/local.ts',
152
- 'src/embeddings/types.ts',
153
- 'src/cli-extractors.ts',
154
- 'src/extraction/typedFacts.ts',
155
- 'src/extraction/eventExtractor.ts',
156
- 'src/extraction/preferenceExtractor.ts',
157
- 'src/eval/hashEmbedding.ts',
158
- 'src/router/sessionRouter.ts',
159
- 'src/digests/digests.ts',
160
- ];
161
- /** Index-code files whose text is not scanned for environment names: orchestrator's flags are all search-time. */
162
- const ENV_SCAN_EXCLUDED = new Set([
163
- 'src/retrieval/orchestrator.ts',
164
- 'src/retrieval/entityLookup.ts',
165
- 'src/retrieval/queryHelpers.ts',
166
- ]);
167
- /** Per-run values and credentials. A credential never decides what is stored, and must never reach a key or a record. */
168
- const ENV_NEVER_KEYED = /(_KEYS?|_KEY_FILE|_SECRET|_PASSWORD|_CREDENTIALS?|_AUTH_TOKEN|_ACCESS_TOKEN|_API_TOKEN)$|^(VAULT_PATH|DB_PATH|PRIORITY_PATHS|HOME|PATH|TMPDIR|PWD)$/;
169
- const sha256 = (data) => node_crypto_1.default.createHash('sha256').update(data).digest('hex');
170
- function listFiles(root, relative) {
171
- const full = node_path_1.default.join(root, relative);
172
- if (!node_fs_1.default.existsSync(full))
173
- throw new Error(`Index code path missing: ${relative}`);
174
- const stat = node_fs_1.default.statSync(full);
175
- if (stat.isFile())
176
- return [relative];
177
- const out = [];
178
- for (const entry of node_fs_1.default.readdirSync(full, { withFileTypes: true })) {
179
- const child = `${relative}/${entry.name}`;
180
- if (entry.isDirectory())
181
- out.push(...listFiles(root, child));
182
- else if (entry.isFile())
183
- out.push(child);
184
- }
185
- return out;
186
- }
187
- /** src/cli.ts up to the `search` command: imports, helpers, setup() and the `index` command; the search handler is read-path. */
188
- function cliIndexHalf(source) {
189
- const index = source.indexOf('.command("index")');
190
- const search = source.indexOf('.command("search');
191
- // If the file is ever reorganised so the index command no longer precedes search, hash the whole file.
192
- return index !== -1 && search > index ? source.slice(0, search) : source;
193
- }
194
- /** The text of `export const NAME ... ;` in a source file. Throws when it is not there: a pinned constant that moved must not drop out of the key. */
195
- function constantDeclaration(text, name, file) {
196
- const match = new RegExp(`export const ${name}\\b[^;]*;`).exec(text);
197
- if (!match)
198
- throw new Error(`Pinned index constant ${name} is not declared as "export const ${name} ...;" in ${file}`);
199
- return match[0];
200
- }
201
- function indexCodeIdentity(repoRoot) {
202
- const files = [...new Set(exports.INDEX_CODE_PATHS.flatMap(entry => listFiles(repoRoot, entry)))]
203
- .filter(file => /\.(ts|json)$/.test(file))
204
- .sort();
205
- const parts = [];
206
- const envNames = new Set();
207
- for (const file of files) {
208
- const bytes = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file));
209
- parts.push(`${file}\0${sha256(bytes)}`);
210
- if (!file.endsWith('.ts') || ENV_SCAN_EXCLUDED.has(file))
211
- continue;
212
- const text = bytes.toString('utf8');
213
- for (const match of text.matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
214
- envNames.add(match[1]);
215
- for (const match of text.matchAll(/\benv\[\s*['"]([A-Z][A-Z0-9_]*)['"]\s*\]/g))
216
- envNames.add(match[1]);
217
- // A name held in a list or passed to a helper (envFlag('RENOR_X')) still names a variable this code reads.
218
- for (const match of text.matchAll(/['"`](RENOR_[A-Z0-9_]+)['"`]/g))
219
- envNames.add(match[1]);
220
- }
221
- for (const [file, names] of Object.entries(exports.INDEX_PINNED_CONSTANTS)) {
222
- const text = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file), 'utf8');
223
- for (const name of names)
224
- parts.push(`${file}#${name}\0${sha256(constantDeclaration(text, name, file))}`);
225
- }
226
- const cli = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'src/cli.ts'), 'utf8');
227
- parts.push(`src/cli.ts#index-half\0${sha256(cliIndexHalf(cli))}`);
228
- for (const match of cliIndexHalf(cli).matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
229
- envNames.add(match[1]);
230
- let lock = '';
231
- try {
232
- lock = sha256(node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'package-lock.json')));
233
- }
234
- catch {
235
- lock = 'no-lockfile';
236
- }
237
- parts.push(`package-lock.json\0${lock}`);
238
- return {
239
- sha256: sha256(parts.join('\n')),
240
- files: files.length + 2,
241
- envNames: [...envNames].filter(name => !ENV_NEVER_KEYED.test(name)).sort(),
242
- };
243
- }
244
- /** A loopback proxy's port is a property of this run, not of the vectors: fold it away so the box and the laptop agree. */
245
- function normalizeEndpoint(value) {
246
- return value.replace(/^(https?):\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d+)?(?=\/|$)/, '$1://loopback');
247
- }
248
- function indexConfig(input) {
249
- const code = indexCodeIdentity(input.repoRoot);
250
- const present = [];
251
- const values = code.envNames.map(name => {
252
- const raw = input.env[name];
253
- if (raw === undefined)
254
- return `${name}=<unset>`;
255
- present.push(name);
256
- return `${name}=${normalizeEndpoint(raw)}`;
257
- });
258
- const components = {
259
- schema: exports.SNAPSHOT_SCHEMA,
260
- codeSha256: code.sha256,
261
- codeFiles: code.files,
262
- envSha256: sha256(values.join('\n')),
263
- envNames: code.envNames,
264
- envSet: present,
265
- embeddingNamespace: input.embeddingNamespace ?? null,
266
- node: `${process.version.split('.')[0]}`,
267
- platform: `${process.platform}/${process.arch}`,
268
- vaultBase: node_path_1.default.resolve(input.vaultBase),
269
- };
270
- return { hash: sha256(JSON.stringify(components)), components };
271
- }
272
- /** What the indexer reads of a rendered haystack: each file's path, bytes and modification time. */
273
- function vaultSignature(vaultDir, files) {
274
- const parts = files.map(file => {
275
- const stat = node_fs_1.default.statSync(node_path_1.default.join(vaultDir, file.relativePath));
276
- return `${file.relativePath}\0${file.sha256}\0${stat.mtimeMs}`;
277
- });
278
- return sha256(parts.join('\n'));
279
- }
280
- function vaultDirFor(vaultBase, questionId) {
281
- // The id names a directory: keep it to a safe segment, and a hash of the id so two ids can never share one.
282
- const segment = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
283
- const root = node_path_1.default.join(node_path_1.default.resolve(vaultBase), segment);
284
- return { root, vaultDir: node_path_1.default.join(root, 'vault') };
285
- }
286
- /**
287
- * The router/digest stats of a restored index. The build that made the snapshot made those model calls; this run
288
- * did not, and a record that repeated the build's counts would overstate this run's calls and tokens. The
289
- * counters this run spent read zero, the notes it did not have to process read as cached, and the build's own
290
- * line is kept whole under builtWith.
291
- */
292
- function restoredReplay(replay) {
293
- const zeroed = (stats, cachedFrom) => ({
294
- ...stats,
295
- ...(cachedFrom && typeof stats[cachedFrom] === 'number' ? { cached: stats[cachedFrom] } : {}),
296
- ...Object.fromEntries(['summarized', 'calls', 'promptTokens', 'completionTokens', 'failed'].filter(key => typeof stats[key] === 'number').map(key => [key, 0])),
297
- restoredFromSnapshot: true,
298
- builtWith: stats,
299
- });
300
- return {
301
- ...(replay?.routerSummaries ? { routerSummaries: zeroed(replay.routerSummaries, 'notes') } : {}),
302
- ...(replay?.digestStats ? { digestStats: zeroed(replay.digestStats, null) } : {}),
303
- };
304
- }
305
- function sha256File(file) {
306
- const hash = node_crypto_1.default.createHash('sha256');
307
- const fd = node_fs_1.default.openSync(file, 'r');
308
- try {
309
- const buffer = Buffer.allocUnsafe(1 << 20);
310
- for (let read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null); read > 0; read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null)) {
311
- hash.update(buffer.subarray(0, read));
312
- }
313
- }
314
- finally {
315
- node_fs_1.default.closeSync(fd);
316
- }
317
- return hash.digest('hex');
318
- }
319
- function processAlive(pid) {
320
- try {
321
- process.kill(pid, 0);
322
- return true;
323
- }
324
- catch (error) {
325
- return error.code === 'EPERM';
326
- }
327
- }
328
- class SnapshotStore {
329
- options;
330
- stats = { restored: 0, built: 0, saved: 0, stale: 0, corrupt: 0, busy: 0, saveFailed: 0, indexMsSaved: 0 };
331
- dir;
332
- constructor(options) {
333
- this.options = options;
334
- this.dir = node_path_1.default.join(node_path_1.default.resolve(options.root), 'snapshots', options.config.hash.slice(0, 16));
335
- }
336
- get config() { return this.options.config; }
337
- files(questionId) {
338
- const name = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
339
- return { db: node_path_1.default.join(this.dir, `${name}.db`), meta: node_path_1.default.join(this.dir, `${name}.json`) };
340
- }
341
- /**
342
- * Take the vault slot of a question: two runs on one machine must not render into the same place at once. A
343
- * lock held by a live process means "build this one the ordinary way" (null).
344
- */
345
- claim(questionId) {
346
- const { root, vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
347
- node_fs_1.default.mkdirSync(node_path_1.default.dirname(root), { recursive: true, mode: 0o700 });
348
- const lockPath = `${root}.lock`;
349
- for (let attempt = 0; attempt < 2; attempt++) {
350
- try {
351
- const fd = node_fs_1.default.openSync(lockPath, 'wx');
352
- node_fs_1.default.writeSync(fd, JSON.stringify({ pid: process.pid, at: new Date().toISOString() }));
353
- node_fs_1.default.closeSync(fd);
354
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
355
- node_fs_1.default.mkdirSync(root, { recursive: true, mode: 0o700 });
356
- return {
357
- root,
358
- vaultDir,
359
- release: () => {
360
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
361
- node_fs_1.default.rmSync(lockPath, { force: true });
362
- },
363
- };
364
- }
365
- catch (error) {
366
- if (error.code !== 'EEXIST')
367
- throw error;
368
- let owner = 0;
369
- try {
370
- owner = JSON.parse(node_fs_1.default.readFileSync(lockPath, 'utf8')).pid ?? 0;
371
- }
372
- catch { /* torn lock: stale */ }
373
- if (owner && processAlive(owner)) {
374
- this.stats.busy++;
375
- return null;
376
- }
377
- node_fs_1.default.rmSync(lockPath, { force: true }); // a crashed owner's lock
378
- }
379
- }
380
- this.stats.busy++;
381
- return null;
382
- }
383
- /** Put the snapshot of a question at dbPath when it is current and intact. */
384
- restore(questionId, signature, dbPath) {
385
- const { db, meta: metaPath } = this.files(questionId);
386
- if (!node_fs_1.default.existsSync(metaPath) && !node_fs_1.default.existsSync(db))
387
- return { state: 'miss' };
388
- let meta;
389
- try {
390
- meta = JSON.parse(node_fs_1.default.readFileSync(metaPath, 'utf8'));
391
- }
392
- catch {
393
- this.stats.corrupt++;
394
- return { state: 'corrupt', reason: 'metadata missing or unreadable' };
395
- }
396
- const { vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
397
- if (meta.schema !== exports.SNAPSHOT_SCHEMA || meta.questionId !== questionId || meta.configHash !== this.options.config.hash
398
- || meta.vaultSignature !== signature || meta.vaultDir !== vaultDir) {
399
- this.stats.stale++;
400
- return { state: 'stale', reason: 'key mismatch' };
401
- }
402
- try {
403
- const stat = node_fs_1.default.statSync(db);
404
- if (stat.size !== meta.dbBytes || sha256File(db) !== meta.dbSha256) {
405
- this.stats.corrupt++;
406
- return { state: 'corrupt', reason: 'snapshot file does not match its recorded hash' };
407
- }
408
- node_fs_1.default.copyFileSync(db, dbPath);
409
- // The copy is what the search will open: it must be the bytes that were verified.
410
- if (sha256File(dbPath) !== meta.dbSha256) {
411
- node_fs_1.default.rmSync(dbPath, { force: true });
412
- this.stats.corrupt++;
413
- return { state: 'corrupt', reason: 'copy does not match the verified snapshot' };
414
- }
415
- }
416
- catch (error) {
417
- node_fs_1.default.rmSync(dbPath, { force: true });
418
- this.stats.corrupt++;
419
- return { state: 'corrupt', reason: error instanceof Error ? error.message : String(error) };
420
- }
421
- this.stats.restored++;
422
- this.stats.indexMsSaved += meta.indexMs;
423
- return { state: 'restored', meta };
424
- }
425
- /** Save the index just built at dbPath. A failure to save never fails the run: the next run builds again. */
426
- save(args) {
427
- this.stats.built++;
428
- const { db, meta: metaPath } = this.files(args.questionId);
429
- try {
430
- // The index holds every haystack statement in plain text: owner-only, as the index itself is.
431
- node_fs_1.default.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
432
- // Fold the write-ahead log into the file; a snapshot is one self-contained file.
433
- const handle = (0, client_1.openDb)(args.dbPath);
434
- try {
435
- handle.pragma('wal_checkpoint(TRUNCATE)');
436
- }
437
- finally {
438
- handle.close();
439
- }
440
- for (const suffix of ['-wal', '-shm']) {
441
- if (node_fs_1.default.existsSync(`${args.dbPath}${suffix}`) && node_fs_1.default.statSync(`${args.dbPath}${suffix}`).size > 0) {
442
- throw new Error(`index still has a ${suffix} file after checkpoint`);
443
- }
444
- }
445
- const tmp = `${db}.${process.pid}.tmp`;
446
- node_fs_1.default.copyFileSync(args.dbPath, tmp);
447
- node_fs_1.default.chmodSync(tmp, 0o600);
448
- const meta = {
449
- schema: exports.SNAPSHOT_SCHEMA,
450
- questionId: args.questionId,
451
- configHash: this.options.config.hash,
452
- vaultSignature: args.signature,
453
- dbSha256: sha256File(tmp),
454
- dbBytes: node_fs_1.default.statSync(tmp).size,
455
- createdAt: new Date().toISOString(),
456
- indexMs: Math.round(args.indexMs),
457
- vaultDir: vaultDirFor(this.options.vaultBase, args.questionId).vaultDir,
458
- ...(args.replay && Object.keys(args.replay).length > 0 ? { indexReplay: args.replay } : {}),
459
- ...(args.gitSha ? { gitSha: args.gitSha } : {}),
460
- components: this.options.config.components,
461
- };
462
- // The metadata is the commit: written last, so a reader never sees metadata without its file.
463
- node_fs_1.default.renameSync(tmp, db);
464
- const metaTmp = `${metaPath}.${process.pid}.tmp`;
465
- node_fs_1.default.writeFileSync(metaTmp, JSON.stringify(meta, null, 2) + '\n', { mode: 0o600 });
466
- node_fs_1.default.renameSync(metaTmp, metaPath);
467
- this.stats.saved++;
468
- return true;
469
- }
470
- catch (error) {
471
- this.stats.saveFailed++;
472
- process.stderr.write(`[snapshots] not saved for ${args.questionId}: ${error instanceof Error ? error.message : String(error)}\n`);
473
- return false;
474
- }
475
- }
476
- }
477
- exports.SnapshotStore = SnapshotStore;
478
- function snapshotReport(root) {
479
- const base = node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots');
480
- const sets = [];
481
- if (node_fs_1.default.existsSync(base)) {
482
- for (const set of node_fs_1.default.readdirSync(base, { withFileTypes: true }).filter(entry => entry.isDirectory())) {
483
- const dir = node_path_1.default.join(base, set.name);
484
- let bytes = 0;
485
- let snapshots = 0;
486
- let oldest = null;
487
- let newest = null;
488
- let built = null;
489
- for (const file of node_fs_1.default.readdirSync(dir)) {
490
- const full = node_path_1.default.join(dir, file);
491
- bytes += node_fs_1.default.statSync(full).size;
492
- if (!file.endsWith('.json'))
493
- continue;
494
- snapshots++;
495
- try {
496
- const meta = JSON.parse(node_fs_1.default.readFileSync(full, 'utf8'));
497
- if (!oldest || meta.createdAt < oldest)
498
- oldest = meta.createdAt;
499
- if (!newest || meta.createdAt > newest)
500
- newest = meta.createdAt;
501
- built ??= { node: meta.components.node, platform: meta.components.platform, vaultBase: meta.components.vaultBase, embeddingNamespace: meta.components.embeddingNamespace };
502
- }
503
- catch { /* a torn metadata file still counts its bytes */ }
504
- }
505
- sets.push({ configHash16: set.name, snapshots, bytes, oldest, newest, built });
506
- }
507
- }
508
- sets.sort((a, b) => (b.newest ?? '').localeCompare(a.newest ?? ''));
509
- return { root: node_path_1.default.resolve(root), totalBytes: sets.reduce((sum, set) => sum + set.bytes, 0), sets };
510
- }
511
- function pruneSnapshots(root, options) {
512
- if (options.olderThanDays === undefined && options.keepLatest === undefined && !(options.configs?.length)) {
513
- throw new Error('prune needs --older-than, --keep-latest or --config: it never removes everything by default');
514
- }
515
- const report = snapshotReport(root);
516
- const now = options.now ?? Date.now();
517
- const doomed = new Set();
518
- report.sets.forEach((set, index) => {
519
- if (options.keepLatest !== undefined && index >= options.keepLatest)
520
- doomed.add(set.configHash16);
521
- if (options.olderThanDays !== undefined && (!set.newest || now - Date.parse(set.newest) > options.olderThanDays * 86_400_000))
522
- doomed.add(set.configHash16);
523
- if (options.configs?.some(prefix => prefix && set.configHash16.startsWith(prefix)))
524
- doomed.add(set.configHash16);
525
- });
526
- const removed = report.sets.filter(set => doomed.has(set.configHash16));
527
- if (!options.dryRun) {
528
- for (const set of removed)
529
- node_fs_1.default.rmSync(node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots', set.configHash16), { recursive: true, force: true });
530
- }
531
- return { removed, bytes: removed.reduce((sum, set) => sum + set.bytes, 0), dryRun: Boolean(options.dryRun) };
532
- }
533
- /**
534
- * Snapshot directory: flag, else RENOR_EVAL_SNAPSHOT_DIR, else `index-snapshots` in the benchmark data directory
535
- * (RENOR_BENCH_DATA), else the user's cache directory. The last default names no machine's own layout.
536
- */
537
- function resolveSnapshotRoot(flag, env = process.env) {
538
- if (flag?.trim())
539
- return node_path_1.default.resolve(flag);
540
- if (env.RENOR_EVAL_SNAPSHOT_DIR?.trim())
541
- return node_path_1.default.resolve(env.RENOR_EVAL_SNAPSHOT_DIR);
542
- if (env.RENOR_BENCH_DATA?.trim())
543
- return node_path_1.default.join(node_path_1.default.resolve(env.RENOR_BENCH_DATA), 'index-snapshots');
544
- return node_path_1.default.join(node_path_1.default.resolve(env.HOME ?? '.'), '.cache', 'renor-eval', 'index-snapshots');
545
- }
546
- function resolveVaultBase(flag, env = process.env) {
547
- return node_path_1.default.resolve(flag?.trim() || env.RENOR_EVAL_SNAPSHOT_VAULT_BASE?.trim() || exports.DEFAULT_VAULT_BASE);
548
- }
@@ -1,56 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseTimestamp = parseTimestamp;
4
- exports.validateLongMemRows = validateLongMemRows;
5
- /** Upstream timestamps have no zone. Interpret them as UTC, never host-local time. */
6
- function parseTimestamp(value) {
7
- const official = /^(\d{4})\/(\d{2})\/(\d{2})(?: \([A-Za-z]{3}\))?(?: (\d{2}):(\d{2}))?$/.exec(value);
8
- const iso = official ? `${official[1]}-${official[2]}-${official[3]}T${official[4] ?? '00'}:${official[5] ?? '00'}:00Z` : value;
9
- if (!/(Z|[+-]\d{2}:\d{2})$/.test(iso))
10
- throw new Error(`Ambiguous timestamp: ${value}`);
11
- const date = new Date(iso);
12
- if (Number.isNaN(date.getTime()) || (official && date.toISOString().slice(0, 16) !== iso.slice(0, 16)))
13
- throw new Error(`Invalid timestamp: ${value}`);
14
- return date;
15
- }
16
- function uniqueStrings(values, label) {
17
- if (!Array.isArray(values) || values.some(v => typeof v !== 'string' || !v.trim()))
18
- throw new Error(`Invalid ${label}`);
19
- if (new Set(values).size !== values.length)
20
- throw new Error(`Duplicate ${label}`);
21
- }
22
- function validateLongMemRows(rows) {
23
- if (!Array.isArray(rows) || !rows.length)
24
- throw new Error('Empty/invalid LongMemEval dataset');
25
- uniqueStrings(rows.map(r => r.question_id), 'question id');
26
- for (const row of rows) {
27
- if (!row.question?.trim() || !row.question_type?.trim())
28
- throw new Error('Missing question/type');
29
- parseTimestamp(row.question_date);
30
- uniqueStrings(row.answer_session_ids, 'answer session id');
31
- // Official data may repeat a noise haystack session id; a repeated ANSWER id
32
- // would make evidence mapping ambiguous and stays rejected.
33
- const seenAnswer = new Set(row.answer_session_ids);
34
- const haystackCounts = new Map();
35
- for (const id of row.haystack_session_ids) {
36
- haystackCounts.set(id, (haystackCounts.get(id) ?? 0) + 1);
37
- if ((haystackCounts.get(id) ?? 0) > 1 && seenAnswer.has(id)) {
38
- throw new Error(`Duplicated answer session id in haystack: ${id}`);
39
- }
40
- }
41
- if (!Array.isArray(row.haystack_sessions) || !Array.isArray(row.haystack_dates)
42
- || row.haystack_sessions.length !== row.haystack_session_ids.length
43
- || row.haystack_dates.length !== row.haystack_session_ids.length)
44
- throw new Error('Haystack arrays differ in length');
45
- row.haystack_dates.forEach(parseTimestamp);
46
- // Official data legitimately contains empty noise sessions; roles/content stay strict.
47
- for (const turns of row.haystack_sessions) {
48
- if (!Array.isArray(turns) || turns.some(t => !['user', 'assistant'].includes(t.role) || typeof t.content !== 'string')) {
49
- throw new Error('Invalid session turns');
50
- }
51
- }
52
- if (!row.question_id.endsWith('_abs') && (!row.answer_session_ids.length
53
- || row.answer_session_ids.some(id => !row.haystack_session_ids.includes(id))))
54
- throw new Error('Missing answer evidence sessions');
55
- }
56
- }