renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,549 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.SnapshotStore = exports.INDEX_ENTRY_MODULES = exports.READ_PATH_ONLY = exports.DYNAMIC_ENV_READS = exports.READ_PATH_BINDINGS = exports.INDEX_PINNED_CONSTANTS = exports.INDEX_CODE_PATHS = exports.DEFAULT_VAULT_BASE = exports.SNAPSHOT_SCHEMA = void 0;
7
- exports.cliIndexHalf = cliIndexHalf;
8
- exports.constantDeclaration = constantDeclaration;
9
- exports.indexCodeIdentity = indexCodeIdentity;
10
- exports.indexConfig = indexConfig;
11
- exports.vaultSignature = vaultSignature;
12
- exports.vaultDirFor = vaultDirFor;
13
- exports.restoredReplay = restoredReplay;
14
- exports.snapshotReport = snapshotReport;
15
- exports.pruneSnapshots = pruneSnapshots;
16
- exports.resolveSnapshotRoot = resolveSnapshotRoot;
17
- exports.resolveVaultBase = resolveVaultBase;
18
- const node_fs_1 = __importDefault(require("node:fs"));
19
- const node_path_1 = __importDefault(require("node:path"));
20
- const node_crypto_1 = __importDefault(require("node:crypto"));
21
- const client_1 = require("../db/client");
22
- /**
23
- * Index snapshots for LongMemEval iteration runs.
24
- *
25
- * The product runner builds one memory store per question (render the haystack as markdown, run `cli index`).
26
- * That store depends only on the question's haystack and on the code and configuration that decide what the
27
- * indexer stores, so a later run that changes only retrieval, page building or reading can restore it instead
28
- * of building it again. A snapshot is the SQLite file `cli index` leaves behind, copied right after the index
29
- * and before any search touches it.
30
- *
31
- * Correctness rests on three things, all enforced here:
32
- * 1. The key is a hash of everything that reaches the stored bytes: the indexing code (INDEX_CODE_PATHS, the
33
- * index half of src/cli.ts, the lockfile), the environment the indexing code reads (scanned from those
34
- * sources, so a new flag is in the key without anyone remembering it), the embedding identity, the Node
35
- * major and platform, the vault location and the rendered haystack itself. Any difference is a miss and the
36
- * store is rebuilt.
37
- * 2. The index stores absolute paths (notes, chunks and their full-text tables), so a snapshot only works at
38
- * the path it was built at. The vault of a snapshotted question is therefore rendered at a fixed place
39
- * (vaultDirFor), independent of where the snapshots themselves live, and that place is part of the key.
40
- * Moving the snapshot directory to another machine works; changing the vault base builds a new set.
41
- * 3. A snapshot is verified (sha-256 of the file against the metadata written after it) before it is used. A
42
- * missing, truncated or altered one is a rebuild, never an error and never a silent use.
43
- *
44
- * Nothing here is used for promotion arms: they build cold (the runner refuses the combination).
45
- */
46
- exports.SNAPSHOT_SCHEMA = 'indexsnap-v1';
47
- /** Where the vault of a snapshotted question is rendered. On a RAM-backed temp directory by default (no SSD writes). */
48
- exports.DEFAULT_VAULT_BASE = '/tmp/renor-index-vault';
49
- /**
50
- * The source that decides what `cli index` stores. Directories are hashed whole. src/retrieval/orchestrator.ts
51
- * is here because the indexer imports three date helpers from it (resolveSessionDate, sessionDateMs,
52
- * boundContentDate); a change to it re-indexes, which is the safe side. Everything else retrieval, page
53
- * building, evidence and the reader is deliberately NOT here: tests/eval/index-snapshots.test.ts walks the
54
- * import graph from the index command and fails when a module appears that is neither listed here nor named in
55
- * READ_PATH_ONLY, so the boundary cannot drift unnoticed.
56
- */
57
- exports.INDEX_CODE_PATHS = [
58
- 'src/indexer',
59
- 'src/db',
60
- 'src/embeddings',
61
- 'src/extraction',
62
- 'src/memory',
63
- 'src/digests',
64
- 'src/router',
65
- 'src/net',
66
- 'src/config.ts',
67
- 'src/cli-extractors.ts',
68
- 'src/context/rowStore.ts',
69
- 'src/eval/hashEmbedding.ts',
70
- 'src/mvp/log.ts',
71
- 'src/mvp/scope.ts',
72
- 'src/toolRefusal.ts',
73
- 'src/evidence/ollamaChat.ts',
74
- 'src/evidence/keyPool.ts',
75
- 'src/evidence/options.ts',
76
- 'src/retrieval/orchestrator.ts',
77
- 'src/retrieval/entityLookup.ts',
78
- 'src/retrieval/queryHelpers.ts',
79
- ];
80
- /**
81
- * Constants the index code takes from a read-path module. The module stays out of the hash (an edit to the evidence
82
- * second pass is a search-time lever and must not re-index), but the declaration of each constant the index code
83
- * reads is hashed: the write-time digest lane and the session router fall back to EVIDENCE_MODEL_DEFAULT when no
84
- * model variable is set, so changing that default changes the digests and summaries a cold build stores. A name that
85
- * is not declared as `export const NAME ...;` in its file is an error, never a skipped entry.
86
- */
87
- exports.INDEX_PINNED_CONSTANTS = {
88
- 'src/evidence/secondPass.ts': ['EVIDENCE_MODEL_DEFAULT', 'EVIDENCE_REASONING_EFFORTS'],
89
- };
90
- /**
91
- * Every other binding that hashed code imports from a READ_PATH_ONLY module, with why it does not decide what an
92
- * index stores. tests/eval/index-snapshots.test.ts compares this with the real imports, so a new binding taken from
93
- * a read-path module fails until someone classifies it: pin it above, or list it here with the reason.
94
- * Keyed `<module>#<binding>`.
95
- */
96
- exports.READ_PATH_BINDINGS = {
97
- 'src/evidence/attach.ts#renderEvidenceTable': 'digests.ts renders a digest table into a page at search time',
98
- 'src/evidence/evidenceCache.ts#EvidenceCache': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
99
- 'src/evidence/optin.ts#EvidenceDailyBudget': 'options.ts resolveEvidenceChat, the search-time evidence pass',
100
- 'src/evidence/optin.ts#optinEndpointAllowed': 'options.ts resolveEvidenceChat, the search-time evidence pass',
101
- 'src/evidence/optin.ts#readOptinKey': 'options.ts resolveEvidenceChat, the search-time evidence pass',
102
- 'src/evidence/optin.ts#resolveEvidenceOptin': 'options.ts resolveEvidenceChat, the search-time evidence pass',
103
- 'src/evidence/optin.ts#dailyBudgetGate': 'options.ts resolveEvidenceChat, the search-time evidence pass',
104
- 'src/evidence/gate.ts#resolveEvidenceMode': 'options.ts resolveEvidenceToolOptions, the search-time evidence pass',
105
- 'src/evidence/secondPass.ts#MAX_EVIDENCE_SESSIONS': 'options.ts evidence bounds, search time',
106
- 'src/evidence/secondPass.ts#EVIDENCE_SESSION_CHAR_CAP': 'options.ts evidence bounds, search time',
107
- 'src/evidence/secondPass.ts#EvidenceExtractor': 'a type, erased at build time',
108
- };
109
- /** Hashed files whose environment reads the lexical scan cannot see by name, with what covers them instead. */
110
- exports.DYNAMIC_ENV_READS = {
111
- 'src/evidence/keyPool.ts': 'reads the Ollama credential variables by a name list; credentials are never in the key',
112
- 'src/memory/statusFacts.ts': 'indexes env[STATUS_FRESHNESS_FLAG], a constant whose RENOR_ literal is in the same file and so in the scan',
113
- };
114
- /**
115
- * Modules the index command can reach through an import but does not run to produce stored state: they are
116
- * imported by orchestrator.ts / digests.ts / the router for their search-time functions. Listed so the import
117
- * walk in the test names every file it does not hash, with the reason.
118
- */
119
- exports.READ_PATH_ONLY = [
120
- 'src/evidence/attach.ts', // digests.ts imports renderEvidenceTable, used when a page is built
121
- 'src/evidence/evidenceCache.ts',
122
- 'src/evidence/gate.ts',
123
- 'src/evidence/optin.ts',
124
- 'src/evidence/secondPass.ts', // router.ts reads the EVIDENCE_MODEL_DEFAULT constant (RENOR_EVIDENCE_MODEL is in the key)
125
- 'src/mvp/contextRelevance.ts', // orchestrator: calibrated relevance bars at search time
126
- 'src/retrieval/conversationShape.ts',
127
- 'src/retrieval/dailyEntryTime.ts',
128
- 'src/retrieval/factsFirst.ts',
129
- 'src/retrieval/keyword.ts',
130
- 'src/retrieval/ledgerEventTime.ts',
131
- 'src/retrieval/ledgerLiveness.ts',
132
- 'src/retrieval/ledgerPresentation.ts',
133
- 'src/retrieval/plainWording.ts',
134
- 'src/retrieval/rankingTrace.ts', // orchestrator records search-time learning diagnostics; tracing is inert while index runs
135
- 'src/retrieval/semantic.ts',
136
- ];
137
- /**
138
- * The modules the `index` command builds on: what src/cli.ts's setup() and index action construct and call. The import
139
- * walk in tests/eval/index-snapshots.test.ts starts here. A new dependency of the index command belongs in this list
140
- * and in INDEX_CODE_PATHS.
141
- */
142
- exports.INDEX_ENTRY_MODULES = [
143
- 'src/config.ts',
144
- 'src/db/client.ts',
145
- 'src/db/schema.ts',
146
- 'src/db/sqlite.ts',
147
- 'src/memory/lifecycle.ts',
148
- 'src/indexer/pipeline.ts',
149
- 'src/indexer/staleness.ts',
150
- 'src/embeddings/vectorIndex.ts',
151
- 'src/embeddings/openai.ts',
152
- 'src/embeddings/local.ts',
153
- 'src/embeddings/types.ts',
154
- 'src/cli-extractors.ts',
155
- 'src/extraction/typedFacts.ts',
156
- 'src/extraction/eventExtractor.ts',
157
- 'src/extraction/preferenceExtractor.ts',
158
- 'src/eval/hashEmbedding.ts',
159
- 'src/router/sessionRouter.ts',
160
- 'src/digests/digests.ts',
161
- ];
162
- /** Index-code files whose text is not scanned for environment names: orchestrator's flags are all search-time. */
163
- const ENV_SCAN_EXCLUDED = new Set([
164
- 'src/retrieval/orchestrator.ts',
165
- 'src/retrieval/entityLookup.ts',
166
- 'src/retrieval/queryHelpers.ts',
167
- ]);
168
- /** Per-run values and credentials. A credential never decides what is stored, and must never reach a key or a record. */
169
- const ENV_NEVER_KEYED = /(_KEYS?|_KEY_FILE|_SECRET|_PASSWORD|_CREDENTIALS?|_AUTH_TOKEN|_ACCESS_TOKEN|_API_TOKEN)$|^(VAULT_PATH|DB_PATH|PRIORITY_PATHS|HOME|PATH|TMPDIR|PWD)$/;
170
- const sha256 = (data) => node_crypto_1.default.createHash('sha256').update(data).digest('hex');
171
- function listFiles(root, relative) {
172
- const full = node_path_1.default.join(root, relative);
173
- if (!node_fs_1.default.existsSync(full))
174
- throw new Error(`Index code path missing: ${relative}`);
175
- const stat = node_fs_1.default.statSync(full);
176
- if (stat.isFile())
177
- return [relative];
178
- const out = [];
179
- for (const entry of node_fs_1.default.readdirSync(full, { withFileTypes: true })) {
180
- const child = `${relative}/${entry.name}`;
181
- if (entry.isDirectory())
182
- out.push(...listFiles(root, child));
183
- else if (entry.isFile())
184
- out.push(child);
185
- }
186
- return out;
187
- }
188
- /** src/cli.ts up to the `search` command: imports, helpers, setup() and the `index` command; the search handler is read-path. */
189
- function cliIndexHalf(source) {
190
- const index = source.indexOf('.command("index")');
191
- const search = source.indexOf('.command("search');
192
- // If the file is ever reorganised so the index command no longer precedes search, hash the whole file.
193
- return index !== -1 && search > index ? source.slice(0, search) : source;
194
- }
195
- /** The text of `export const NAME ... ;` in a source file. Throws when it is not there: a pinned constant that moved must not drop out of the key. */
196
- function constantDeclaration(text, name, file) {
197
- const match = new RegExp(`export const ${name}\\b[^;]*;`).exec(text);
198
- if (!match)
199
- throw new Error(`Pinned index constant ${name} is not declared as "export const ${name} ...;" in ${file}`);
200
- return match[0];
201
- }
202
- function indexCodeIdentity(repoRoot) {
203
- const files = [...new Set(exports.INDEX_CODE_PATHS.flatMap(entry => listFiles(repoRoot, entry)))]
204
- .filter(file => /\.(ts|json)$/.test(file))
205
- .sort();
206
- const parts = [];
207
- const envNames = new Set();
208
- for (const file of files) {
209
- const bytes = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file));
210
- parts.push(`${file}\0${sha256(bytes)}`);
211
- if (!file.endsWith('.ts') || ENV_SCAN_EXCLUDED.has(file))
212
- continue;
213
- const text = bytes.toString('utf8');
214
- for (const match of text.matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
215
- envNames.add(match[1]);
216
- for (const match of text.matchAll(/\benv\[\s*['"]([A-Z][A-Z0-9_]*)['"]\s*\]/g))
217
- envNames.add(match[1]);
218
- // A name held in a list or passed to a helper (envFlag('RENOR_X')) still names a variable this code reads.
219
- for (const match of text.matchAll(/['"`](RENOR_[A-Z0-9_]+)['"`]/g))
220
- envNames.add(match[1]);
221
- }
222
- for (const [file, names] of Object.entries(exports.INDEX_PINNED_CONSTANTS)) {
223
- const text = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, file), 'utf8');
224
- for (const name of names)
225
- parts.push(`${file}#${name}\0${sha256(constantDeclaration(text, name, file))}`);
226
- }
227
- const cli = node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'src/cli.ts'), 'utf8');
228
- parts.push(`src/cli.ts#index-half\0${sha256(cliIndexHalf(cli))}`);
229
- for (const match of cliIndexHalf(cli).matchAll(/\benv\.([A-Z][A-Z0-9_]*)/g))
230
- envNames.add(match[1]);
231
- let lock = '';
232
- try {
233
- lock = sha256(node_fs_1.default.readFileSync(node_path_1.default.join(repoRoot, 'package-lock.json')));
234
- }
235
- catch {
236
- lock = 'no-lockfile';
237
- }
238
- parts.push(`package-lock.json\0${lock}`);
239
- return {
240
- sha256: sha256(parts.join('\n')),
241
- files: files.length + 2,
242
- envNames: [...envNames].filter(name => !ENV_NEVER_KEYED.test(name)).sort(),
243
- };
244
- }
245
- /** A loopback proxy's port is a property of this run, not of the vectors: fold it away so the box and the laptop agree. */
246
- function normalizeEndpoint(value) {
247
- return value.replace(/^(https?):\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d+)?(?=\/|$)/, '$1://loopback');
248
- }
249
- function indexConfig(input) {
250
- const code = indexCodeIdentity(input.repoRoot);
251
- const present = [];
252
- const values = code.envNames.map(name => {
253
- const raw = input.env[name];
254
- if (raw === undefined)
255
- return `${name}=<unset>`;
256
- present.push(name);
257
- return `${name}=${normalizeEndpoint(raw)}`;
258
- });
259
- const components = {
260
- schema: exports.SNAPSHOT_SCHEMA,
261
- codeSha256: code.sha256,
262
- codeFiles: code.files,
263
- envSha256: sha256(values.join('\n')),
264
- envNames: code.envNames,
265
- envSet: present,
266
- embeddingNamespace: input.embeddingNamespace ?? null,
267
- node: `${process.version.split('.')[0]}`,
268
- platform: `${process.platform}/${process.arch}`,
269
- vaultBase: node_path_1.default.resolve(input.vaultBase),
270
- };
271
- return { hash: sha256(JSON.stringify(components)), components };
272
- }
273
- /** What the indexer reads of a rendered haystack: each file's path, bytes and modification time. */
274
- function vaultSignature(vaultDir, files) {
275
- const parts = files.map(file => {
276
- const stat = node_fs_1.default.statSync(node_path_1.default.join(vaultDir, file.relativePath));
277
- return `${file.relativePath}\0${file.sha256}\0${stat.mtimeMs}`;
278
- });
279
- return sha256(parts.join('\n'));
280
- }
281
- function vaultDirFor(vaultBase, questionId) {
282
- // The id names a directory: keep it to a safe segment, and a hash of the id so two ids can never share one.
283
- const segment = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
284
- const root = node_path_1.default.join(node_path_1.default.resolve(vaultBase), segment);
285
- return { root, vaultDir: node_path_1.default.join(root, 'vault') };
286
- }
287
- /**
288
- * The router/digest stats of a restored index. The build that made the snapshot made those model calls; this run
289
- * did not, and a record that repeated the build's counts would overstate this run's calls and tokens. The
290
- * counters this run spent read zero, the notes it did not have to process read as cached, and the build's own
291
- * line is kept whole under builtWith.
292
- */
293
- function restoredReplay(replay) {
294
- const zeroed = (stats, cachedFrom) => ({
295
- ...stats,
296
- ...(cachedFrom && typeof stats[cachedFrom] === 'number' ? { cached: stats[cachedFrom] } : {}),
297
- ...Object.fromEntries(['summarized', 'calls', 'promptTokens', 'completionTokens', 'failed'].filter(key => typeof stats[key] === 'number').map(key => [key, 0])),
298
- restoredFromSnapshot: true,
299
- builtWith: stats,
300
- });
301
- return {
302
- ...(replay?.routerSummaries ? { routerSummaries: zeroed(replay.routerSummaries, 'notes') } : {}),
303
- ...(replay?.digestStats ? { digestStats: zeroed(replay.digestStats, null) } : {}),
304
- };
305
- }
306
- function sha256File(file) {
307
- const hash = node_crypto_1.default.createHash('sha256');
308
- const fd = node_fs_1.default.openSync(file, 'r');
309
- try {
310
- const buffer = Buffer.allocUnsafe(1 << 20);
311
- for (let read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null); read > 0; read = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, null)) {
312
- hash.update(buffer.subarray(0, read));
313
- }
314
- }
315
- finally {
316
- node_fs_1.default.closeSync(fd);
317
- }
318
- return hash.digest('hex');
319
- }
320
- function processAlive(pid) {
321
- try {
322
- process.kill(pid, 0);
323
- return true;
324
- }
325
- catch (error) {
326
- return error.code === 'EPERM';
327
- }
328
- }
329
- class SnapshotStore {
330
- options;
331
- stats = { restored: 0, built: 0, saved: 0, stale: 0, corrupt: 0, busy: 0, saveFailed: 0, indexMsSaved: 0 };
332
- dir;
333
- constructor(options) {
334
- this.options = options;
335
- this.dir = node_path_1.default.join(node_path_1.default.resolve(options.root), 'snapshots', options.config.hash.slice(0, 16));
336
- }
337
- get config() { return this.options.config; }
338
- files(questionId) {
339
- const name = `${questionId.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80)}-${sha256(questionId).slice(0, 8)}`;
340
- return { db: node_path_1.default.join(this.dir, `${name}.db`), meta: node_path_1.default.join(this.dir, `${name}.json`) };
341
- }
342
- /**
343
- * Take the vault slot of a question: two runs on one machine must not render into the same place at once. A
344
- * lock held by a live process means "build this one the ordinary way" (null).
345
- */
346
- claim(questionId) {
347
- const { root, vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
348
- node_fs_1.default.mkdirSync(node_path_1.default.dirname(root), { recursive: true, mode: 0o700 });
349
- const lockPath = `${root}.lock`;
350
- for (let attempt = 0; attempt < 2; attempt++) {
351
- try {
352
- const fd = node_fs_1.default.openSync(lockPath, 'wx');
353
- node_fs_1.default.writeSync(fd, JSON.stringify({ pid: process.pid, at: new Date().toISOString() }));
354
- node_fs_1.default.closeSync(fd);
355
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
356
- node_fs_1.default.mkdirSync(root, { recursive: true, mode: 0o700 });
357
- return {
358
- root,
359
- vaultDir,
360
- release: () => {
361
- node_fs_1.default.rmSync(root, { recursive: true, force: true });
362
- node_fs_1.default.rmSync(lockPath, { force: true });
363
- },
364
- };
365
- }
366
- catch (error) {
367
- if (error.code !== 'EEXIST')
368
- throw error;
369
- let owner = 0;
370
- try {
371
- owner = JSON.parse(node_fs_1.default.readFileSync(lockPath, 'utf8')).pid ?? 0;
372
- }
373
- catch { /* torn lock: stale */ }
374
- if (owner && processAlive(owner)) {
375
- this.stats.busy++;
376
- return null;
377
- }
378
- node_fs_1.default.rmSync(lockPath, { force: true }); // a crashed owner's lock
379
- }
380
- }
381
- this.stats.busy++;
382
- return null;
383
- }
384
- /** Put the snapshot of a question at dbPath when it is current and intact. */
385
- restore(questionId, signature, dbPath) {
386
- const { db, meta: metaPath } = this.files(questionId);
387
- if (!node_fs_1.default.existsSync(metaPath) && !node_fs_1.default.existsSync(db))
388
- return { state: 'miss' };
389
- let meta;
390
- try {
391
- meta = JSON.parse(node_fs_1.default.readFileSync(metaPath, 'utf8'));
392
- }
393
- catch {
394
- this.stats.corrupt++;
395
- return { state: 'corrupt', reason: 'metadata missing or unreadable' };
396
- }
397
- const { vaultDir } = vaultDirFor(this.options.vaultBase, questionId);
398
- if (meta.schema !== exports.SNAPSHOT_SCHEMA || meta.questionId !== questionId || meta.configHash !== this.options.config.hash
399
- || meta.vaultSignature !== signature || meta.vaultDir !== vaultDir) {
400
- this.stats.stale++;
401
- return { state: 'stale', reason: 'key mismatch' };
402
- }
403
- try {
404
- const stat = node_fs_1.default.statSync(db);
405
- if (stat.size !== meta.dbBytes || sha256File(db) !== meta.dbSha256) {
406
- this.stats.corrupt++;
407
- return { state: 'corrupt', reason: 'snapshot file does not match its recorded hash' };
408
- }
409
- node_fs_1.default.copyFileSync(db, dbPath);
410
- // The copy is what the search will open: it must be the bytes that were verified.
411
- if (sha256File(dbPath) !== meta.dbSha256) {
412
- node_fs_1.default.rmSync(dbPath, { force: true });
413
- this.stats.corrupt++;
414
- return { state: 'corrupt', reason: 'copy does not match the verified snapshot' };
415
- }
416
- }
417
- catch (error) {
418
- node_fs_1.default.rmSync(dbPath, { force: true });
419
- this.stats.corrupt++;
420
- return { state: 'corrupt', reason: error instanceof Error ? error.message : String(error) };
421
- }
422
- this.stats.restored++;
423
- this.stats.indexMsSaved += meta.indexMs;
424
- return { state: 'restored', meta };
425
- }
426
- /** Save the index just built at dbPath. A failure to save never fails the run: the next run builds again. */
427
- save(args) {
428
- this.stats.built++;
429
- const { db, meta: metaPath } = this.files(args.questionId);
430
- try {
431
- // The index holds every haystack statement in plain text: owner-only, as the index itself is.
432
- node_fs_1.default.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
433
- // Fold the write-ahead log into the file; a snapshot is one self-contained file.
434
- const handle = (0, client_1.openDb)(args.dbPath);
435
- try {
436
- handle.pragma('wal_checkpoint(TRUNCATE)');
437
- }
438
- finally {
439
- handle.close();
440
- }
441
- for (const suffix of ['-wal', '-shm']) {
442
- if (node_fs_1.default.existsSync(`${args.dbPath}${suffix}`) && node_fs_1.default.statSync(`${args.dbPath}${suffix}`).size > 0) {
443
- throw new Error(`index still has a ${suffix} file after checkpoint`);
444
- }
445
- }
446
- const tmp = `${db}.${process.pid}.tmp`;
447
- node_fs_1.default.copyFileSync(args.dbPath, tmp);
448
- node_fs_1.default.chmodSync(tmp, 0o600);
449
- const meta = {
450
- schema: exports.SNAPSHOT_SCHEMA,
451
- questionId: args.questionId,
452
- configHash: this.options.config.hash,
453
- vaultSignature: args.signature,
454
- dbSha256: sha256File(tmp),
455
- dbBytes: node_fs_1.default.statSync(tmp).size,
456
- createdAt: new Date().toISOString(),
457
- indexMs: Math.round(args.indexMs),
458
- vaultDir: vaultDirFor(this.options.vaultBase, args.questionId).vaultDir,
459
- ...(args.replay && Object.keys(args.replay).length > 0 ? { indexReplay: args.replay } : {}),
460
- ...(args.gitSha ? { gitSha: args.gitSha } : {}),
461
- components: this.options.config.components,
462
- };
463
- // The metadata is the commit: written last, so a reader never sees metadata without its file.
464
- node_fs_1.default.renameSync(tmp, db);
465
- const metaTmp = `${metaPath}.${process.pid}.tmp`;
466
- node_fs_1.default.writeFileSync(metaTmp, JSON.stringify(meta, null, 2) + '\n', { mode: 0o600 });
467
- node_fs_1.default.renameSync(metaTmp, metaPath);
468
- this.stats.saved++;
469
- return true;
470
- }
471
- catch (error) {
472
- this.stats.saveFailed++;
473
- process.stderr.write(`[snapshots] not saved for ${args.questionId}: ${error instanceof Error ? error.message : String(error)}\n`);
474
- return false;
475
- }
476
- }
477
- }
478
- exports.SnapshotStore = SnapshotStore;
479
- function snapshotReport(root) {
480
- const base = node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots');
481
- const sets = [];
482
- if (node_fs_1.default.existsSync(base)) {
483
- for (const set of node_fs_1.default.readdirSync(base, { withFileTypes: true }).filter(entry => entry.isDirectory())) {
484
- const dir = node_path_1.default.join(base, set.name);
485
- let bytes = 0;
486
- let snapshots = 0;
487
- let oldest = null;
488
- let newest = null;
489
- let built = null;
490
- for (const file of node_fs_1.default.readdirSync(dir)) {
491
- const full = node_path_1.default.join(dir, file);
492
- bytes += node_fs_1.default.statSync(full).size;
493
- if (!file.endsWith('.json'))
494
- continue;
495
- snapshots++;
496
- try {
497
- const meta = JSON.parse(node_fs_1.default.readFileSync(full, 'utf8'));
498
- if (!oldest || meta.createdAt < oldest)
499
- oldest = meta.createdAt;
500
- if (!newest || meta.createdAt > newest)
501
- newest = meta.createdAt;
502
- built ??= { node: meta.components.node, platform: meta.components.platform, vaultBase: meta.components.vaultBase, embeddingNamespace: meta.components.embeddingNamespace };
503
- }
504
- catch { /* a torn metadata file still counts its bytes */ }
505
- }
506
- sets.push({ configHash16: set.name, snapshots, bytes, oldest, newest, built });
507
- }
508
- }
509
- sets.sort((a, b) => (b.newest ?? '').localeCompare(a.newest ?? ''));
510
- return { root: node_path_1.default.resolve(root), totalBytes: sets.reduce((sum, set) => sum + set.bytes, 0), sets };
511
- }
512
- function pruneSnapshots(root, options) {
513
- if (options.olderThanDays === undefined && options.keepLatest === undefined && !(options.configs?.length)) {
514
- throw new Error('prune needs --older-than, --keep-latest or --config: it never removes everything by default');
515
- }
516
- const report = snapshotReport(root);
517
- const now = options.now ?? Date.now();
518
- const doomed = new Set();
519
- report.sets.forEach((set, index) => {
520
- if (options.keepLatest !== undefined && index >= options.keepLatest)
521
- doomed.add(set.configHash16);
522
- if (options.olderThanDays !== undefined && (!set.newest || now - Date.parse(set.newest) > options.olderThanDays * 86_400_000))
523
- doomed.add(set.configHash16);
524
- if (options.configs?.some(prefix => prefix && set.configHash16.startsWith(prefix)))
525
- doomed.add(set.configHash16);
526
- });
527
- const removed = report.sets.filter(set => doomed.has(set.configHash16));
528
- if (!options.dryRun) {
529
- for (const set of removed)
530
- node_fs_1.default.rmSync(node_path_1.default.join(node_path_1.default.resolve(root), 'snapshots', set.configHash16), { recursive: true, force: true });
531
- }
532
- return { removed, bytes: removed.reduce((sum, set) => sum + set.bytes, 0), dryRun: Boolean(options.dryRun) };
533
- }
534
- /**
535
- * Snapshot directory: flag, else RENOR_EVAL_SNAPSHOT_DIR, else `index-snapshots` in the benchmark data directory
536
- * (RENOR_BENCH_DATA), else the user's cache directory. The last default names no machine's own layout.
537
- */
538
- function resolveSnapshotRoot(flag, env = process.env) {
539
- if (flag?.trim())
540
- return node_path_1.default.resolve(flag);
541
- if (env.RENOR_EVAL_SNAPSHOT_DIR?.trim())
542
- return node_path_1.default.resolve(env.RENOR_EVAL_SNAPSHOT_DIR);
543
- if (env.RENOR_BENCH_DATA?.trim())
544
- return node_path_1.default.join(node_path_1.default.resolve(env.RENOR_BENCH_DATA), 'index-snapshots');
545
- return node_path_1.default.join(node_path_1.default.resolve(env.HOME ?? '.'), '.cache', 'renor-eval', 'index-snapshots');
546
- }
547
- function resolveVaultBase(flag, env = process.env) {
548
- return node_path_1.default.resolve(flag?.trim() || env.RENOR_EVAL_SNAPSHOT_VAULT_BASE?.trim() || exports.DEFAULT_VAULT_BASE);
549
- }
@@ -1,56 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseTimestamp = parseTimestamp;
4
- exports.validateLongMemRows = validateLongMemRows;
5
- /** Upstream timestamps have no zone. Interpret them as UTC, never host-local time. */
6
- function parseTimestamp(value) {
7
- const official = /^(\d{4})\/(\d{2})\/(\d{2})(?: \([A-Za-z]{3}\))?(?: (\d{2}):(\d{2}))?$/.exec(value);
8
- const iso = official ? `${official[1]}-${official[2]}-${official[3]}T${official[4] ?? '00'}:${official[5] ?? '00'}:00Z` : value;
9
- if (!/(Z|[+-]\d{2}:\d{2})$/.test(iso))
10
- throw new Error(`Ambiguous timestamp: ${value}`);
11
- const date = new Date(iso);
12
- if (Number.isNaN(date.getTime()) || (official && date.toISOString().slice(0, 16) !== iso.slice(0, 16)))
13
- throw new Error(`Invalid timestamp: ${value}`);
14
- return date;
15
- }
16
- function uniqueStrings(values, label) {
17
- if (!Array.isArray(values) || values.some(v => typeof v !== 'string' || !v.trim()))
18
- throw new Error(`Invalid ${label}`);
19
- if (new Set(values).size !== values.length)
20
- throw new Error(`Duplicate ${label}`);
21
- }
22
- function validateLongMemRows(rows) {
23
- if (!Array.isArray(rows) || !rows.length)
24
- throw new Error('Empty/invalid LongMemEval dataset');
25
- uniqueStrings(rows.map(r => r.question_id), 'question id');
26
- for (const row of rows) {
27
- if (!row.question?.trim() || !row.question_type?.trim())
28
- throw new Error('Missing question/type');
29
- parseTimestamp(row.question_date);
30
- uniqueStrings(row.answer_session_ids, 'answer session id');
31
- // Official data may repeat a noise haystack session id; a repeated ANSWER id
32
- // would make evidence mapping ambiguous and stays rejected.
33
- const seenAnswer = new Set(row.answer_session_ids);
34
- const haystackCounts = new Map();
35
- for (const id of row.haystack_session_ids) {
36
- haystackCounts.set(id, (haystackCounts.get(id) ?? 0) + 1);
37
- if ((haystackCounts.get(id) ?? 0) > 1 && seenAnswer.has(id)) {
38
- throw new Error(`Duplicated answer session id in haystack: ${id}`);
39
- }
40
- }
41
- if (!Array.isArray(row.haystack_sessions) || !Array.isArray(row.haystack_dates)
42
- || row.haystack_sessions.length !== row.haystack_session_ids.length
43
- || row.haystack_dates.length !== row.haystack_session_ids.length)
44
- throw new Error('Haystack arrays differ in length');
45
- row.haystack_dates.forEach(parseTimestamp);
46
- // Official data legitimately contains empty noise sessions; roles/content stay strict.
47
- for (const turns of row.haystack_sessions) {
48
- if (!Array.isArray(turns) || turns.some(t => !['user', 'assistant'].includes(t.role) || typeof t.content !== 'string')) {
49
- throw new Error('Invalid session turns');
50
- }
51
- }
52
- if (!row.question_id.endsWith('_abs') && (!row.answer_session_ids.length
53
- || row.answer_session_ids.some(id => !row.haystack_session_ids.includes(id))))
54
- throw new Error('Missing answer evidence sessions');
55
- }
56
- }