renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,549 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.SnapshotStore = exports.INDEX_ENTRY_MODULES = exports.READ_PATH_ONLY = exports.DYNAMIC_ENV_READS = exports.READ_PATH_BINDINGS = exports.INDEX_PINNED_CONSTANTS = exports.INDEX_CODE_PATHS = exports.DEFAULT_VAULT_BASE = exports.SNAPSHOT_SCHEMA = void 0;
7
- exports.cliIndexHalf = cliIndexHalf;
8
- exports.constantDeclaration = constantDeclaration;
9
- exports.indexCodeIdentity = indexCodeIdentity;
10
- exports.indexConfig = indexConfig;
11
- exports.vaultSignature = vaultSignature;
12
- exports.vaultDirFor = vaultDirFor;
13
- exports.restoredReplay = restoredReplay;
14
- exports.snapshotReport = snapshotReport;
15
- exports.pruneSnapshots = pruneSnapshots;
16
- exports.resolveSnapshotRoot = resolveSnapshotRoot;
17
- exports.resolveVaultBase = resolveVaultBase;
18
- const node_fs_1 = __importDefault(require("node:fs"));
19
- const node_path_1 = __importDefault(require("node:path"));
20
- const node_crypto_1 = __importDefault(require("node:crypto"));
21
- const client_1 = require("../db/client");
22
- /**
23
- * Index snapshots for LongMemEval iteration runs.
24
- *
25
- * The product runner builds one memory store per question (render the haystack as markdown, run `cli index`).
26
- * That store depends only on the question's haystack and on the code and configuration that decide what the
27
- * indexer stores, so a later run that changes only retrieval, page building or reading can restore it instead
28
- * of building it again. A snapshot is the SQLite file `cli index` leaves behind, copied right after the index
29
- * and before any search touches it.
30
- *
31
- * Correctness rests on three things, all enforced here:
32
- * 1. The key is a hash of everything that reaches the stored bytes: the indexing code (INDEX_CODE_PATHS, the
33
- * index half of src/cli.ts, the lockfile), the environment the indexing code reads (scanned from those
34
- * sources, so a new flag is in the key without anyone remembering it), the embedding identity, the Node
35
- * major and platform, the vault location and the rendered haystack itself. Any difference is a miss and the
36
- * store is rebuilt.
37
- * 2. The index stores absolute paths (notes, chunks and their full-text tables), so a snapshot only works at
38
- * the path it was built at. The vault of a snapshotted question is therefore rendered at a fixed place
39
- * (vaultDirFor), independent of where the snapshots themselves live, and that place is part of the key.
40
- * Moving the snapshot directory to another machine works; changing the vault base builds a new set.
41
- * 3. A snapshot is verified (sha-256 of the file against the metadata written after it) before it is used. A
42
- * missing, truncated or altered one is a rebuild, never an error and never a silent use.
43
- *
44
- * Nothing here is used for promotion arms: they build cold (the runner refuses the combination).
45
- */
46
- exports.SNAPSHOT_SCHEMA = 'indexsnap-v1';
47
- /** Where the vault of a snapshotted question is rendered. On a RAM-backed temp directory by default (no SSD writes). */
48
- exports.DEFAULT_VAULT_BASE = '/tmp/renor-index-vault';
49
- /**
50
- * The source that decides what `cli index` stores. Directories are hashed whole. src/retrieval/orchestrator.ts
51
- * is here because the indexer imports three date helpers from it (resolveSessionDate, sessionDateMs,
52
- * boundContentDate); a change to it re-indexes, which is the safe side. Everything else retrieval, page
53
- * building, evidence and the reader is deliberately NOT here: tests/eval/index-snapshots.test.ts walks the
54
- * import graph from the index command and fails when a module appears that is neither listed here nor named in
55
- * READ_PATH_ONLY, so the boundary cannot drift unnoticed.
56
- */
57
- exports.INDEX_CODE_PATHS = [
58
- 'src/indexer',
59
- 'src/db',
60
- 'src/embeddings',
61
- 'src/extraction',
62
- 'src/memory',
63
- 'src/digests',
64
- 'src/router',
65
- 'src/net',
66
- 'src/config.ts',
67
- 'src/cli-extractors.ts',
68
- 'src/context/rowStore.ts',
69
- 'src/eval/hashEmbedding.ts',
70
- 'src/mvp/log.ts',
71
- 'src/mvp/scope.ts',
72
- 'src/toolRefusal.ts',
73
- 'src/evidence/ollamaChat.ts',
74
- 'src/evidence/keyPool.ts',
75
- 'src/evidence/options.ts',
76
- 'src/retrieval/orchestrator.ts',
77
- 'src/retrieval/entityLookup.ts',
78
- 'src/retrieval/queryHelpers.ts',
79
- ];
80
- /**
81
- * Constants the index code takes from a read-path module. The module stays out of the hash (an edit to the evidence
82
- * second pass is a search-time lever and must not re-index), but the declaration of each constant the index code
83
- * reads is hashed: the write-time digest lane and the session router fall back to EVIDENCE_MODEL_DEFAULT when no
84
- * model variable is set, so changing that default changes the digests and summaries a cold build stores. A name that
85
- * is not declared as `export const NAME ...;` in its file is an error, never a skipped entry.
86
- */
87
- exports.INDEX_PINNED_CONSTANTS = {
88
- 'src/evidence/secondPass.ts': ['EVIDENCE_MODEL_DEFAULT', 'EVIDENCE_REASONING_EFFORTS'],
89
- };
90
- /**
91
- * Every other binding that hashed code imports from a READ_PATH_ONLY module, with why it does not decide what an
92
- * index stores. tests/eval/index-snapshots.test.ts compares this with the real imports, so a new binding taken from
93
- * a read-path module fails until someone classifies it: pin it above, or list it here with the reason.
94
- * Keyed `<module>#<binding>`.
95
- */
96
- exports.READ_PATH_BINDINGS = {
97
- 'src/evidence/attach.ts#renderEvidenceTable': 'digests.ts renders a digest table into a page at search time',
98
- 'src/evidence/evidenceCache.ts#EvidenceCache': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
99
- 'src/evidence/optin.ts#EvidenceDailyBudget': 'options.ts resolveEvidenceChat, the search-time evidence pass',
100
- 'src/evidence/optin.ts#optinEndpointAllowed': 'options.ts resolveEvidenceChat, the search-time evidence pass',
101
- 'src/evidence/optin.ts#readOptinKey': 'options.ts resolveEvidenceChat, the search-time evidence pass',
102
- 'src/evidence/optin.ts#resolveEvidenceOptin': 'options.ts resolveEvidenceChat, the search-time evidence pass',
103
- 'src/evidence/optin.ts#dailyBudgetGate': 'options.ts resolveEvidenceChat, the search-time evidence pass',
104
- 'src/evidence/gate.ts#resolveEvidenceMode': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
105
- 'src/evidence/secondPass.ts#MAX_EVIDENCE_SESSIONS': 'options.ts evidence bounds, search time',
106
- 'src/evidence/secondPass.ts#EVIDENCE_SESSION_CHAR_CAP': 'options.ts evidence bounds, search time',
107
- 'src/evidence/secondPass.ts#EvidenceExtractor': 'a type, erased at build time',
108
- };
109
- /** Hashed files whose environment reads the lexical scan cannot see by name, with what covers them instead. */
110
- exports.DYNAMIC_ENV_READS = {
111
- 'src/evidence/keyPool.ts': 'reads the Ollama credential variables by a name list; credentials are never in the key',
112
- 'src/memory/statusFacts.ts': 'indexes env[STATUS_FRESHNESS_FLAG], a constant whose RENOR_ literal is in the same file and so in the scan',
113
- };
114
- /**
115
- * Modules the index command can reach through an import but does not run to produce stored state: they are
116
- * imported by orchestrator.ts / digests.ts / the router for their search-time functions. Listed so the import
117
- * walk in the test names every file it does not hash, with the reason.
118
- */
119
- exports.READ_PATH_ONLY = [
120
- 'src/evidence/attach.ts', // digests.ts imports renderEvidenceTable, used when a page is built
121
- 'src/evidence/evidenceCache.ts',
122
- 'src/evidence/gate.ts',
123
- 'src/evidence/optin.ts',
124
- 'src/evidence/secondPass.ts', // router.ts reads the EVIDENCE_MODEL_DEFAULT constant (RENOR_EVIDENCE_MODEL is in the key)
125
- 'src/mvp/contextRelevance.ts', // orchestrator: calibrated relevance bars at search time
126
- 'src/retrieval/conversationShape.ts',
127
- 'src/retrieval/dailyEntryTime.ts',
128
- 'src/retrieval/factsFirst.ts',
129
- 'src/retrieval/keyword.ts',
130
- 'src/retrieval/ledgerEventTime.ts',
131
- 'src/retrieval/ledgerLiveness.ts',
132
- 'src/retrieval/ledgerPresentation.ts',
133
- 'src/retrieval/plainWording.ts',
134
- 'src/retrieval/rankingTrace.ts', // orchestrator records search-time learning diagnostics; tracing is inert while index runs
135
- 'src/retrieval/semantic.ts',
136
- ];
137
- /**
138
- * The modules the `index` command builds on: what src/cli.ts's setup() and index action construct and call. The import
139
- * walk in tests/eval/index-snapshots.test.ts starts here. A new dependency of the index command belongs in this list
140
- * and in INDEX_CODE_PATHS.
141
- */
142
- exports.INDEX_ENTRY_MODULES = [
143
- 'src/config.ts',
144
- 'src/db/client.ts',
145
- 'src/db/schema.ts',
146
- 'src/db/sqlite.ts',
147
- 'src/memory/lifecycle.ts',
148
- 'src/indexer/pipeline.ts',
149
- 'src/indexer/staleness.ts',
150
- 'src/embeddings/vectorIndex.ts',
151
- 'src/embeddings/openai.ts',
152
- 'src/embeddings/local.ts',
153
- 'src/embeddings/types.ts',
154
- 'src/cli-extractors.ts',
155
- 'src/extraction/typedFacts.ts',
156
- 'src/extraction/eventExtractor.ts',
157
- 'src/extraction/preferenceExtractor.ts',
158
- 'src/eval/hashEmbedding.ts',
159
- 'src/router/sessionRouter.ts',
160
- 'src/digests/digests.ts',
161
- ];
162
- /** Index-code files whose text is not scanned for environment names: orchestrator's flags are all search-time. */
163
- const ENV_SCAN_EXCLUDED = new Set([
164
- 'src/retrieval/orchestrator.ts',
165
- 'src/retrieval/entityLookup.ts',
166
- 'src/retrieval/queryHelpers.ts',
167
- ]);
168
- /** Per-run values and credentials. A credential never decides what is stored, and must never reach a key or a record. */
169
- const ENV_NEVER_KEYED = /(_KEYS?|_KEY_FILE|_SECRET|_PASSWORD|_CREDENTIALS?|_AUTH_TOKEN|_ACCESS_TOKEN|_API_TOKEN)$|^(VAULT_PATH|DB_PATH|PRIORITY_PATHS|HOME|PATH|TMPDIR|PWD)$/;
170
- const sha256 = (data) => node_crypto_1.default.createHash('sha256').update(data).digest('hex');
171
- function listFiles(root, relative) {
172
- const full = node_path_1.default.join(root, relative);
173
- if (!node_fs_1.default.existsSync(full))
174
- throw new Error(`Index code path missing: ${relative}`);
175
- const stat = node_fs_1.default.statSync(full);
176
- if (stat.isFile())
177
- return [relative];
178
- const out = [];
179
- for (const entry of node_fs_1.default.readdirSync(full, { withFileTypes: true })) {
180
- const child = `${relative}/${entry.name}`;
181
- if (entry.isDirectory())
182
- out.push(...listFiles(root, child));
183
- else if (entry.isFile())
184
- out.push(child);
185
- }
186
- return out;
187
- }
188
- /** src/cli.ts up to the `search` command: imports, helpers, setup() and the `index` command; the search handler is read-path. */
189
- function cliIndexHalf(source) {
190
- const index = source.indexOf('.command("index")');
191
- const search = source.indexOf('.command("search');
192
- // If the file is ever reorganised so the index command no longer precedes search, hash the whole file.
193
- return index !== -1 && search > index ? source.slice(0, search) : source;
194
- }
195
- /** The text of `export const NAME ... ;` in a source file. Throws when it is not there: a pinned constant that moved must not drop out of the key. */
196
- function constantDeclaration(text, name, file) {
197
- const match = new RegExp(`export const ${name}\\b[^;]*;`).exec(text);
198
- if (!match)
199
- throw new Error(`Pinned index constant ${name} is not declared as "export const ${name} ...;" in ${file}`);
200
- return match[0];
201
- }
202
- function indexCodeIdentity(repoRoot) {
203
- const files = [...new Set(exports.INDEX_CODE_PATHS.flatMap(entry => listFiles(repoRoot, entry)))]
204
- .filter(file => /\.(ts|json)$/.test(file))
205
- .sort();
206
- const parts = [];
207
- const envNames = new Set();
208
- for (const file of files) {
209
- const bytes = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file));
210
- parts.push(`${file}\0${sha256(bytes)}`);
211
- if (!file.endsWith('.ts') || ENV_SCAN_EXCLUDED.has(file))
212
- continue;
213
- const text = bytes.toString('utf8');
214
- for (const match of text.matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
215
- envNames.add(match[1]);
216
- for (const match of text.matchAll(/\benv\[\s*['"]([A-Z][A-Z0-9_]*)['"]\s*\]/g))
217
- envNames.add(match[1]);
218
- // A name held in a list or passed to a helper (envFlag('RENOR_X')) still names a variable this code reads.
219
- for (const match of text.matchAll(/['"`](RENOR_[A-Z0-9_]+)['"`]/g))
220
- envNames.add(match[1]);
221
- }
222
- for (const [file, names] of Object.entries(exports.INDEX_PINNED_CONSTANTS)) {
223
- const text = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file), 'utf8');
224
- for (const name of names)
225
- parts.push(`${file}#${name}\0${sha256(constantDeclaration(text, name, file))}`);
226
- }
227
- const cli = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'src/cli.ts'), 'utf8');
228
- parts.push(`src/cli.ts#index-half\0${sha256(cliIndexHalf(cli))}`);
229
- for (const match of cliIndexHalf(cli).matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
230
- envNames.add(match[1]);
231
- let lock = '';
232
- try {
233
- lock = sha256(node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'package-lock.json')));
234
- }
235
- catch {
236
- lock = 'no-lockfile';
237
- }
238
- parts.push(`package-lock.json\0${lock}`);
239
- return {
240
- sha256: sha256(parts.join('\n')),
241
- files: files.length + 2,
242
- envNames: [...envNames].filter(name => !ENV_NEVER_KEYED.test(name)).sort(),
243
- };
244
- }
245
- /** A loopback proxy's port is a property of this run, not of the vectors: fold it away so the box and the laptop agree. */
246
- function normalizeEndpoint(value) {
247
- return value.replace(/^(https?):\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d+)?(?=\/|$)/, '$1://loopback');
248
- }
249
- function indexConfig(input) {
250
- const code = indexCodeIdentity(input.repoRoot);
251
- const present = [];
252
- const values = code.envNames.map(name => {
253
- const raw = input.env[name];
254
- if (raw === undefined)
255
- return `${name}=<unset>`;
256
- present.push(name);
257
- return `${name}=${normalizeEndpoint(raw)}`;
258
- });
259
- const components = {
260
- schema: exports.SNAPSHOT_SCHEMA,
261
- codeSha256: code.sha256,
262
- codeFiles: code.files,
263
- envSha256: sha256(values.join('\n')),
264
- envNames: code.envNames,
265
- envSet: present,
266
- embeddingNamespace: input.embeddingNamespace ?? null,
267
- node: `${process.version.split('.')[0]}`,
268
- platform: `${process.platform}/${process.arch}`,
269
- vaultBase: node_path_1.default.resolve(input.vaultBase),
270
- };
271
- return { hash: sha256(JSON.stringify(components)), components };
272
- }
273
- /** What the indexer reads of a rendered haystack: each file's path, bytes and modification time. */
274
- function vaultSignature(vaultDir, files) {
275
- const parts = files.map(file => {
276
- const stat = node_fs_1.default.statSync(node_path_1.default.join(vaultDir, file.relativePath));
277
- return `${file.relativePath}\0${file.sha256}\0${stat.mtimeMs}`;
278
- });
279
- return sha256(parts.join('\n'));
280
- }
281
- function vaultDirFor(vaultBase, questionId) {
282
- // The id names a directory: keep it to a safe segment, and a hash of the id so two ids can never share one.
283
- const segment = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
284
- const root = node_path_1.default.join(node_path_1.default.resolve(vaultBase), segment);
285
- return { root, vaultDir: node_path_1.default.join(root, 'vault') };
286
- }
287
- /**
288
- * The router/digest stats of a restored index. The build that made the snapshot made those model calls; this run
289
- * did not, and a record that repeated the build's counts would overstate this run's calls and tokens. The
290
- * counters this run spent read zero, the notes it did not have to process read as cached, and the build's own
291
- * line is kept whole under builtWith.
292
- */
293
- function restoredReplay(replay) {
294
- const zeroed = (stats, cachedFrom) => ({
295
- ...stats,
296
- ...(cachedFrom && typeof stats[cachedFrom] === 'number' ? { cached: stats[cachedFrom] } : {}),
297
- ...Object.fromEntries(['summarized', 'calls', 'promptTokens', 'completionTokens', 'failed'].filter(key => typeof stats[key] === 'number').map(key => [key, 0])),
298
- restoredFromSnapshot: true,
299
- builtWith: stats,
300
- });
301
- return {
302
- ...(replay?.routerSummaries ? { routerSummaries: zeroed(replay.routerSummaries, 'notes') } : {}),
303
- ...(replay?.digestStats ? { digestStats: zeroed(replay.digestStats, null) } : {}),
304
- };
305
- }
306
- function sha256File(file) {
307
- const hash = node_crypto_1.default.createHash('sha256');
308
- const fd = node_fs_1.default.openSync(file, 'r');
309
- try {
310
- const buffer = Buffer.allocUnsafe(1 << 20);
311
- for (let read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null); read > 0; read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null)) {
312
- hash.update(buffer.subarray(0, read));
313
- }
314
- }
315
- finally {
316
- node_fs_1.default.closeSync(fd);
317
- }
318
- return hash.digest('hex');
319
- }
320
- function processAlive(pid) {
321
- try {
322
- process.kill(pid, 0);
323
- return true;
324
- }
325
- catch (error) {
326
- return error.code === 'EPERM';
327
- }
328
- }
329
- class SnapshotStore {
330
- options;
331
- stats = { restored: 0, built: 0, saved: 0, stale: 0, corrupt: 0, busy: 0, saveFailed: 0, indexMsSaved: 0 };
332
- dir;
333
- constructor(options) {
334
- this.options = options;
335
- this.dir = node_path_1.default.join(node_path_1.default.resolve(options.root), 'snapshots', options.config.hash.slice(0, 16));
336
- }
337
- get config() { return this.options.config; }
338
- files(questionId) {
339
- const name = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
340
- return { db: node_path_1.default.join(this.dir, `${name}.db`), meta: node_path_1.default.join(this.dir, `${name}.json`) };
341
- }
342
- /**
343
- * Take the vault slot of a question: two runs on one machine must not render into the same place at once. A
344
- * lock held by a live process means "build this one the ordinary way" (null).
345
- */
346
- claim(questionId) {
347
- const { root, vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
348
- node_fs_1.default.mkdirSync(node_path_1.default.dirname(root), { recursive: true, mode: 0o700 });
349
- const lockPath = `${root}.lock`;
350
- for (let attempt = 0; attempt < 2; attempt++) {
351
- try {
352
- const fd = node_fs_1.default.openSync(lockPath, 'wx');
353
- node_fs_1.default.writeSync(fd, JSON.stringify({ pid: process.pid, at: new Date().toISOString() }));
354
- node_fs_1.default.closeSync(fd);
355
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
356
- node_fs_1.default.mkdirSync(root, { recursive: true, mode: 0o700 });
357
- return {
358
- root,
359
- vaultDir,
360
- release: () => {
361
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
362
- node_fs_1.default.rmSync(lockPath, { force: true });
363
- },
364
- };
365
- }
366
- catch (error) {
367
- if (error.code !== 'EEXIST')
368
- throw error;
369
- let owner = 0;
370
- try {
371
- owner = JSON.parse(node_fs_1.default.readFileSync(lockPath, 'utf8')).pid ?? 0;
372
- }
373
- catch { /* torn lock: stale */ }
374
- if (owner && processAlive(owner)) {
375
- this.stats.busy++;
376
- return null;
377
- }
378
- node_fs_1.default.rmSync(lockPath, { force: true }); // a crashed owner's lock
379
- }
380
- }
381
- this.stats.busy++;
382
- return null;
383
- }
384
- /** Put the snapshot of a question at dbPath when it is current and intact. */
385
- restore(questionId, signature, dbPath) {
386
- const { db, meta: metaPath } = this.files(questionId);
387
- if (!node_fs_1.default.existsSync(metaPath) && !node_fs_1.default.existsSync(db))
388
- return { state: 'miss' };
389
- let meta;
390
- try {
391
- meta = JSON.parse(node_fs_1.default.readFileSync(metaPath, 'utf8'));
392
- }
393
- catch {
394
- this.stats.corrupt++;
395
- return { state: 'corrupt', reason: 'metadata missing or unreadable' };
396
- }
397
- const { vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
398
- if (meta.schema !== exports.SNAPSHOT_SCHEMA || meta.questionId !== questionId || meta.configHash !== this.options.config.hash
399
- || meta.vaultSignature !== signature || meta.vaultDir !== vaultDir) {
400
- this.stats.stale++;
401
- return { state: 'stale', reason: 'key mismatch' };
402
- }
403
- try {
404
- const stat = node_fs_1.default.statSync(db);
405
- if (stat.size !== meta.dbBytes || sha256File(db) !== meta.dbSha256) {
406
- this.stats.corrupt++;
407
- return { state: 'corrupt', reason: 'snapshot file does not match its recorded hash' };
408
- }
409
- node_fs_1.default.copyFileSync(db, dbPath);
410
- // The copy is what the search will open: it must be the bytes that were verified.
411
- if (sha256File(dbPath) !== meta.dbSha256) {
412
- node_fs_1.default.rmSync(dbPath, { force: true });
413
- this.stats.corrupt++;
414
- return { state: 'corrupt', reason: 'copy does not match the verified snapshot' };
415
- }
416
- }
417
- catch (error) {
418
- node_fs_1.default.rmSync(dbPath, { force: true });
419
- this.stats.corrupt++;
420
- return { state: 'corrupt', reason: error instanceof Error ? error.message : String(error) };
421
- }
422
- this.stats.restored++;
423
- this.stats.indexMsSaved += meta.indexMs;
424
- return { state: 'restored', meta };
425
- }
426
- /** Save the index just built at dbPath. A failure to save never fails the run: the next run builds again. */
427
- save(args) {
428
- this.stats.built++;
429
- const { db, meta: metaPath } = this.files(args.questionId);
430
- try {
431
- // The index holds every haystack statement in plain text: owner-only, as the index itself is.
432
- node_fs_1.default.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
433
- // Fold the write-ahead log into the file; a snapshot is one self-contained file.
434
- const handle = (0, client_1.openDb)(args.dbPath);
435
- try {
436
- handle.pragma('wal_checkpoint(TRUNCATE)');
437
- }
438
- finally {
439
- handle.close();
440
- }
441
- for (const suffix of ['-wal', '-shm']) {
442
- if (node_fs_1.default.existsSync(`${args.dbPath}${suffix}`) && node_fs_1.default.statSync(`${args.dbPath}${suffix}`).size > 0) {
443
- throw new Error(`index still has a ${suffix} file after checkpoint`);
444
- }
445
- }
446
- const tmp = `${db}.${process.pid}.tmp`;
447
- node_fs_1.default.copyFileSync(args.dbPath, tmp);
448
- node_fs_1.default.chmodSync(tmp, 0o600);
449
- const meta = {
450
- schema: exports.SNAPSHOT_SCHEMA,
451
- questionId: args.questionId,
452
- configHash: this.options.config.hash,
453
- vaultSignature: args.signature,
454
- dbSha256: sha256File(tmp),
455
- dbBytes: node_fs_1.default.statSync(tmp).size,
456
- createdAt: new Date().toISOString(),
457
- indexMs: Math.round(args.indexMs),
458
- vaultDir: vaultDirFor(this.options.vaultBase, args.questionId).vaultDir,
459
- ...(args.replay && Object.keys(args.replay).length > 0 ? { indexReplay: args.replay } : {}),
460
- ...(args.gitSha ? { gitSha: args.gitSha } : {}),
461
- components: this.options.config.components,
462
- };
463
- // The metadata is the commit: written last, so a reader never sees metadata without its file.
464
- node_fs_1.default.renameSync(tmp, db);
465
- const metaTmp = `${metaPath}.${process.pid}.tmp`;
466
- node_fs_1.default.writeFileSync(metaTmp, JSON.stringify(meta, null, 2) + '\n', { mode: 0o600 });
467
- node_fs_1.default.renameSync(metaTmp, metaPath);
468
- this.stats.saved++;
469
- return true;
470
- }
471
- catch (error) {
472
- this.stats.saveFailed++;
473
- process.stderr.write(`[snapshots] not saved for ${args.questionId}: ${error instanceof Error ? error.message : String(error)}\n`);
474
- return false;
475
- }
476
- }
477
- }
478
- exports.SnapshotStore = SnapshotStore;
479
- function snapshotReport(root) {
480
- const base = node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots');
481
- const sets = [];
482
- if (node_fs_1.default.existsSync(base)) {
483
- for (const set of node_fs_1.default.readdirSync(base, { withFileTypes: true }).filter(entry => entry.isDirectory())) {
484
- const dir = node_path_1.default.join(base, set.name);
485
- let bytes = 0;
486
- let snapshots = 0;
487
- let oldest = null;
488
- let newest = null;
489
- let built = null;
490
- for (const file of node_fs_1.default.readdirSync(dir)) {
491
- const full = node_path_1.default.join(dir, file);
492
- bytes += node_fs_1.default.statSync(full).size;
493
- if (!file.endsWith('.json'))
494
- continue;
495
- snapshots++;
496
- try {
497
- const meta = JSON.parse(node_fs_1.default.readFileSync(full, 'utf8'));
498
- if (!oldest || meta.createdAt < oldest)
499
- oldest = meta.createdAt;
500
- if (!newest || meta.createdAt > newest)
501
- newest = meta.createdAt;
502
- built ??= { node: meta.components.node, platform: meta.components.platform, vaultBase: meta.components.vaultBase, embeddingNamespace: meta.components.embeddingNamespace };
503
- }
504
- catch { /* a torn metadata file still counts its bytes */ }
505
- }
506
- sets.push({ configHash16: set.name, snapshots, bytes, oldest, newest, built });
507
- }
508
- }
509
- sets.sort((a, b) => (b.newest ?? '').localeCompare(a.newest ?? ''));
510
- return { root: node_path_1.default.resolve(root), totalBytes: sets.reduce((sum, set) => sum + set.bytes, 0), sets };
511
- }
512
- function pruneSnapshots(root, options) {
513
- if (options.olderThanDays === undefined && options.keepLatest === undefined && !(options.configs?.length)) {
514
- throw new Error('prune needs --older-than, --keep-latest or --config: it never removes everything by default');
515
- }
516
- const report = snapshotReport(root);
517
- const now = options.now ?? Date.now();
518
- const doomed = new Set();
519
- report.sets.forEach((set, index) => {
520
- if (options.keepLatest !== undefined && index >= options.keepLatest)
521
- doomed.add(set.configHash16);
522
- if (options.olderThanDays !== undefined && (!set.newest || now - Date.parse(set.newest) > options.olderThanDays * 86_400_000))
523
- doomed.add(set.configHash16);
524
- if (options.configs?.some(prefix => prefix && set.configHash16.startsWith(prefix)))
525
- doomed.add(set.configHash16);
526
- });
527
- const removed = report.sets.filter(set => doomed.has(set.configHash16));
528
- if (!options.dryRun) {
529
- for (const set of removed)
530
- node_fs_1.default.rmSync(node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots', set.configHash16), { recursive: true, force: true });
531
- }
532
- return { removed, bytes: removed.reduce((sum, set) => sum + set.bytes, 0), dryRun: Boolean(options.dryRun) };
533
- }
534
- /**
535
- * Snapshot directory: flag, else RENOR_EVAL_SNAPSHOT_DIR, else `index-snapshots` in the benchmark data directory
536
- * (RENOR_BENCH_DATA), else the user's cache directory. The last default names no machine's own layout.
537
- */
538
- function resolveSnapshotRoot(flag, env = process.env) {
539
- if (flag?.trim())
540
- return node_path_1.default.resolve(flag);
541
- if (env.RENOR_EVAL_SNAPSHOT_DIR?.trim())
542
- return node_path_1.default.resolve(env.RENOR_EVAL_SNAPSHOT_DIR);
543
- if (env.RENOR_BENCH_DATA?.trim())
544
- return node_path_1.default.join(node_path_1.default.resolve(env.RENOR_BENCH_DATA), 'index-snapshots');
545
- return node_path_1.default.join(node_path_1.default.resolve(env.HOME ?? '.'), '.cache', 'renor-eval', 'index-snapshots');
546
- }
547
- function resolveVaultBase(flag, env = process.env) {
548
- return node_path_1.default.resolve(flag?.trim() || env.RENOR_EVAL_SNAPSHOT_VAULT_BASE?.trim() || exports.DEFAULT_VAULT_BASE);
549
- }
@@ -1,56 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseTimestamp = parseTimestamp;
4
- exports.validateLongMemRows = validateLongMemRows;
5
- /** Upstream timestamps have no zone. Interpret them as UTC, never host-local time. */
6
- function parseTimestamp(value) {
7
- const official = /^(\d{4})\/(\d{2})\/(\d{2})(?: \([A-Za-z]{3}\))?(?: (\d{2}):(\d{2}))?$/.exec(value);
8
- const iso = official ? `${official[1]}-${official[2]}-${official[3]}T${official[4] ?? '00'}:${official[5] ?? '00'}:00Z` : value;
9
- if (!/(Z|[+-]\d{2}:\d{2})$/.test(iso))
10
- throw new Error(`Ambiguous timestamp: ${value}`);
11
- const date = new Date(iso);
12
- if (Number.isNaN(date.getTime()) || (official && date.toISOString().slice(0, 16) !== iso.slice(0, 16)))
13
- throw new Error(`Invalid timestamp: ${value}`);
14
- return date;
15
- }
16
- function uniqueStrings(values, label) {
17
- if (!Array.isArray(values) || values.some(v => typeof v !== 'string' || !v.trim()))
18
- throw new Error(`Invalid ${label}`);
19
- if (new Set(values).size !== values.length)
20
- throw new Error(`Duplicate ${label}`);
21
- }
22
- function validateLongMemRows(rows) {
23
- if (!Array.isArray(rows) || !rows.length)
24
- throw new Error('Empty/invalid LongMemEval dataset');
25
- uniqueStrings(rows.map(r => r.question_id), 'question id');
26
- for (const row of rows) {
27
- if (!row.question?.trim() || !row.question_type?.trim())
28
- throw new Error('Missing question/type');
29
- parseTimestamp(row.question_date);
30
- uniqueStrings(row.answer_session_ids, 'answer session id');
31
- // Official data may repeat a noise haystack session id; a repeated ANSWER id
32
- // would make evidence mapping ambiguous and stays rejected.
33
- const seenAnswer = new Set(row.answer_session_ids);
34
- const haystackCounts = new Map();
35
- for (const id of row.haystack_session_ids) {
36
- haystackCounts.set(id, (haystackCounts.get(id) ?? 0) + 1);
37
- if ((haystackCounts.get(id) ?? 0) > 1 && seenAnswer.has(id)) {
38
- throw new Error(`Duplicated answer session id in haystack: ${id}`);
39
- }
40
- }
41
- if (!Array.isArray(row.haystack_sessions) || !Array.isArray(row.haystack_dates)
42
- || row.haystack_sessions.length !== row.haystack_session_ids.length
43
- || row.haystack_dates.length !== row.haystack_session_ids.length)
44
- throw new Error('Haystack arrays differ in length');
45
- row.haystack_dates.forEach(parseTimestamp);
46
- // Official data legitimately contains empty noise sessions; roles/content stay strict.
47
- for (const turns of row.haystack_sessions) {
48
- if (!Array.isArray(turns) || turns.some(t => !['user', 'assistant'].includes(t.role) || typeof t.content !== 'string')) {
49
- throw new Error('Invalid session turns');
50
- }
51
- }
52
- if (!row.question_id.endsWith('_abs') && (!row.answer_session_ids.length
53
- || row.answer_session_ids.some(id => !row.haystack_session_ids.includes(id))))
54
- throw new Error('Missing answer evidence sessions');
55
- }
56
- }