renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -6,8 +6,33 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.VertexEmbeddingProvider = void 0;
7
7
  const node_fs_1 = __importDefault(require("node:fs"));
8
8
  const node_crypto_1 = __importDefault(require("node:crypto"));
9
+ const callPhases_1 = require("../callPhases");
9
10
  const types_1 = require("./types");
10
- const TOKEN_MARGIN_MS = 5 * 60 * 1000;
11
+ /**
12
+ * How a token is kept fresh. Google's lives an hour. A call does not use one with a twelfth of its life left (5 of the 60
13
+ * minutes, the margin it always kept), and the token is renewed in the background when a sixth is left (10 minutes), so the
14
+ * calls keep finding one in hand and none of them waits for the exchange. Shares, not minutes, so a token of any lifetime is
15
+ * treated alike.
16
+ */
17
+ const REQUEST_MARGIN_SHARE = 1 / 12;
18
+ const RENEW_AHEAD_SHARE = 1 / 6;
19
+ /**
20
+ * A token that lives less than this is not renewed in the background: the calls exchange for one when they need it. Google's
21
+ * lives an hour; a service that hands out tokens of seconds would otherwise be asked for one every few seconds for good.
22
+ */
23
+ const MIN_RENEWED_LIFETIME_MS = 60_000;
24
+ /** A renewal that failed is tried again after this, for as long as the token in hand can still be used. */
25
+ const RENEW_RETRY_MS = 60_000;
26
+ /** The exchange is one small request, and every call that needs a token waits for it: a stalled one must not hold them all. */
27
+ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000;
28
+ /** The longest wait setTimeout takes (2^31 - 1 ms); a longer one fires at once. */
29
+ const MAX_TIMER_MS = 2 ** 31 - 1;
30
+ /** The process's own, looked up at each use, so a global clock someone else has replaced is the one it reads. */
31
+ const processClock = {
32
+ now: () => Date.now(),
33
+ setTimeout: (callback, ms) => setTimeout(callback, ms),
34
+ clearTimeout: timer => clearTimeout(timer),
35
+ };
11
36
  /** Quota (429) and transient (500/503) answers are retried: 8 attempts, waiting 10 s, 20 s ... up to 120 s between them. */
12
37
  const DEFAULT_MAX_ATTEMPTS = 8;
13
38
  const defaultRetryWaitMs = (attempt) => Math.min(120_000, 5_000 * 2 ** attempt) + Math.random() * 2_000;
@@ -40,6 +65,14 @@ function sleepUnlessAborted(ms, signal) {
40
65
  function b64url(input) {
41
66
  return Buffer.from(input).toString('base64url');
42
67
  }
68
+ /** `work`, or the signal's reason as soon as it aborts: the caller stops waiting and `work` goes on for the others. */
69
+ function untilAborted(work, signal) {
70
+ return new Promise((resolve, reject) => {
71
+ const onAbort = () => reject(signal.reason);
72
+ signal.addEventListener('abort', onAbort, { once: true });
73
+ work.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
74
+ });
75
+ }
43
76
  class VertexEmbeddingProvider {
44
77
  dimensions;
45
78
  kind = 'vertex';
@@ -53,8 +86,16 @@ class VertexEmbeddingProvider {
53
86
  base;
54
87
  maxAttempts;
55
88
  retryWaitMs;
89
+ /** Where its requests go: the global `fetch`, looked up at each call, unless the caller gave it another (see `request`). */
90
+ request;
91
+ tokenClock;
56
92
  accessToken = null;
57
- tokenExpiresAt = 0;
93
+ /** Epoch ms until which a call may use the token in hand (REQUEST_MARGIN_SHARE), and when it is renewed in the background. */
94
+ tokenUsableUntil = 0;
95
+ tokenRenewAt = 0;
96
+ /** The exchange in flight: a caller that needs a token while one is being fetched waits for it instead of making another. */
97
+ exchange = null;
98
+ renewTimer;
58
99
  usage;
59
100
  constructor(options) {
60
101
  const keyFile = options?.keyFile
@@ -94,13 +135,60 @@ class VertexEmbeddingProvider {
94
135
  this.base = endpointBase(options?.baseUrl, this.location);
95
136
  this.maxAttempts = options?.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
96
137
  this.retryWaitMs = options?.retry?.waitMs ?? defaultRetryWaitMs;
138
+ this.request = options?.request ?? ((input, init) => globalThis.fetch(input, init));
139
+ this.tokenClock = options?.tokenClock ?? processClock;
97
140
  this.usage = { requests: 0, inputTokens: 0, maxInputTokens };
98
141
  }
142
+ /**
143
+ * The token a call uses: the one in hand while it is usable, otherwise the exchange (one at a time, shared by every caller
144
+ * waiting for it). A caller that gives up (its signal aborts) stops waiting; the exchange goes on and the next call finds
145
+ * its token. The background renewal (armRenewal) keeps this from happening on a call at all while the provider is in use.
146
+ */
147
+ getAccessToken(signal) {
148
+ if (this.accessToken && this.tokenClock.now() < this.tokenUsableUntil)
149
+ return Promise.resolve(this.accessToken);
150
+ if (signal?.aborted)
151
+ return Promise.reject(signal.reason); // as before: a call already stopped starts no exchange
152
+ return signal ? untilAborted(this.fetchToken(), signal) : this.fetchToken();
153
+ }
154
+ fetchToken() {
155
+ this.exchange ??= this.exchangeToken().finally(() => { this.exchange = null; });
156
+ return this.exchange;
157
+ }
158
+ /**
159
+ * The token is renewed at RENEW_AHEAD_SHARE of its life left, whether or not a call is waiting for it, and the renewal
160
+ * arms the next one. A timer that cannot wait that long (setTimeout's limit) wakes early and waits again. The timer is
161
+ * unref'd: it never keeps a process alive. A provider that is no longer used goes on renewing while its process lives,
162
+ * one small request about every 50 minutes.
163
+ */
164
+ armRenewal(at, lifetimeMs) {
165
+ this.tokenClock.clearTimeout(this.renewTimer);
166
+ if (!Number.isFinite(at) || !(lifetimeMs >= MIN_RENEWED_LIFETIME_MS))
167
+ return;
168
+ this.tokenRenewAt = at;
169
+ this.waitToRenew(at - this.tokenClock.now());
170
+ }
171
+ waitToRenew(ms) {
172
+ this.renewTimer = this.tokenClock.setTimeout(() => void this.renew(), Math.min(Math.max(0, ms), MAX_TIMER_MS));
173
+ this.renewTimer.unref();
174
+ }
175
+ async renew() {
176
+ const early = this.tokenRenewAt - this.tokenClock.now();
177
+ if (early > 0)
178
+ return this.waitToRenew(early);
179
+ try {
180
+ await this.fetchToken();
181
+ }
182
+ catch {
183
+ // A call exchanges for a token when it must, and reports why if it cannot. Meanwhile, try again while the token in
184
+ // hand can still be used, so a service that was away for a moment is not found out by a call.
185
+ if (this.tokenClock.now() < this.tokenUsableUntil)
186
+ this.waitToRenew(RENEW_RETRY_MS);
187
+ }
188
+ }
99
189
  /** OAuth2 JWT-bearer grant: SA key -> short-lived access token. */
100
- async getAccessToken(signal) {
101
- if (this.accessToken && Date.now() < this.tokenExpiresAt - TOKEN_MARGIN_MS)
102
- return this.accessToken;
103
- const now = Math.floor(Date.now() / 1000);
190
+ async exchangeToken() {
191
+ const now = Math.floor(Date.now() / 1000); // the assertion is dated by the real clock, which is what Google reads
104
192
  const header = b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
105
193
  const claims = b64url(JSON.stringify({
106
194
  iss: this.key.client_email,
@@ -112,20 +200,24 @@ class VertexEmbeddingProvider {
112
200
  const signature = b64url(node_crypto_1.default.createSign('RSA-SHA256')
113
201
  .update(`${header}.${claims}`)
114
202
  .sign(this.key.private_key));
115
- const response = await fetch(this.key.token_uri, {
203
+ const response = await this.request(this.key.token_uri, {
116
204
  method: 'POST',
117
205
  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
118
206
  body: new URLSearchParams({
119
207
  grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
120
208
  assertion: `${header}.${claims}.${signature}`,
121
209
  }).toString(),
122
- signal,
210
+ // Not a caller's signal: the exchange is shared, and one caller giving up must not end it for the others.
211
+ signal: AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS),
123
212
  });
124
213
  if (!response.ok)
125
214
  throw new Error(`Vertex token exchange failed: ${response.status}`);
126
215
  const data = await response.json();
216
+ const lifetimeMs = data.expires_in * 1000;
217
+ const issuedAt = this.tokenClock.now();
127
218
  this.accessToken = data.access_token;
128
- this.tokenExpiresAt = Date.now() + data.expires_in * 1000;
219
+ this.tokenUsableUntil = issuedAt + lifetimeMs * (1 - REQUEST_MARGIN_SHARE);
220
+ this.armRenewal(issuedAt + lifetimeMs * (1 - RENEW_AHEAD_SHARE), lifetimeMs);
129
221
  return this.accessToken;
130
222
  }
131
223
  /**
@@ -134,7 +226,9 @@ class VertexEmbeddingProvider {
134
226
  * deadline. Without one the retries run their whole course.
135
227
  */
136
228
  async predict(contents, call) {
229
+ const waitedForToken = performance.now();
137
230
  const token = await this.getAccessToken(call.signal);
231
+ (0, callPhases_1.recordPhase)('token', performance.now() - waitedForToken);
138
232
  const url = `${this.base}/v1/projects/${this.project}/locations/${this.location}/publishers/google/models/${this.model}:predict`;
139
233
  const body = JSON.stringify({
140
234
  instances: contents.map(({ text, taskType }) => ({
@@ -149,8 +243,15 @@ class VertexEmbeddingProvider {
149
243
  const maxAttempts = this.maxAttempts;
150
244
  let attempt = 0;
151
245
  let response;
246
+ // The request itself, as the log's `call` phase has it: from sending to the last byte read, less the time spent opening a
247
+ // connection for it (which the pool reports as `connect`).
248
+ let sentAt = 0;
249
+ let connectedBefore = 0;
250
+ const spent = () => performance.now() - sentAt - ((0, callPhases_1.recordedPhase)('connect') - connectedBefore);
152
251
  for (;;) {
153
- response = await fetch(url, {
252
+ sentAt = performance.now();
253
+ connectedBefore = (0, callPhases_1.recordedPhase)('connect');
254
+ response = await this.request(url, {
154
255
  method: 'POST',
155
256
  headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
156
257
  body,
@@ -158,6 +259,7 @@ class VertexEmbeddingProvider {
158
259
  });
159
260
  if (response.ok || ![429, 500, 503].includes(response.status))
160
261
  break;
262
+ (0, callPhases_1.recordPhase)('call', spent());
161
263
  attempt += 1;
162
264
  if (attempt >= maxAttempts) {
163
265
  throw new types_1.EmbeddingUnavailableError(`Vertex predict failed after ${attempt} attempts (HTTP ${response.status})`);
@@ -171,10 +273,12 @@ class VertexEmbeddingProvider {
171
273
  }
172
274
  if (!response.ok) {
173
275
  const bodyText = await response.text();
276
+ (0, callPhases_1.recordPhase)('call', spent());
174
277
  // The status travels with the error (the message is unchanged): a 400 names an input the service refuses.
175
278
  throw Object.assign(new Error(`Vertex predict failed (${response.status}): ${bodyText.slice(0, 400)}`), { status: response.status });
176
279
  }
177
280
  const data = await response.json();
281
+ (0, callPhases_1.recordPhase)('call', spent());
178
282
  const predictions = data.predictions ?? [];
179
283
  if (predictions.length !== contents.length
180
284
  || predictions.some(p => !Array.isArray(p.embeddings?.values) || p.embeddings.values.length !== this.dimensions)) {
@@ -196,9 +300,15 @@ class VertexEmbeddingProvider {
196
300
  signal: options?.signal,
197
301
  };
198
302
  const out = [];
199
- for (let i = 0; i < texts.length; i += this.maxInstancesPerRequest) {
200
- const batch = texts.slice(i, i + this.maxInstancesPerRequest).map(text => ({ text, taskType }));
201
- out.push(...await this.predict(batch, call));
303
+ const started = performance.now();
304
+ try {
305
+ for (let i = 0; i < texts.length; i += this.maxInstancesPerRequest) {
306
+ const batch = texts.slice(i, i + this.maxInstancesPerRequest).map(text => ({ text, taskType }));
307
+ out.push(...await this.predict(batch, call));
308
+ }
309
+ }
310
+ finally {
311
+ (0, callPhases_1.recordPhase)('embed', performance.now() - started);
202
312
  }
203
313
  return out;
204
314
  }
@@ -52,9 +52,14 @@ function buildCacheReport(input) {
52
52
  const { answerHits, answerMisses, judgeHits, judgeMisses } = input.evalCache;
53
53
  report.answer = rate({ hits: answerHits, misses: answerMisses });
54
54
  report.judge = rate({ hits: judgeHits, misses: judgeMisses });
55
+ if ((input.evalCache.judgeMemoHits ?? 0) + (input.evalCache.judgeMemoMisses ?? 0) > 0) {
56
+ report.judgeMemo = rate({ hits: input.evalCache.judgeMemoHits ?? 0, misses: input.evalCache.judgeMemoMisses ?? 0 });
57
+ }
55
58
  if (!input.pinnedScope && answerHits + answerMisses + judgeHits + judgeMisses > 0)
56
59
  report.repeatRunBlockers.push(exports.UNPINNED_SCOPE_BLOCKER);
57
60
  }
61
+ if (input.indexSnapshots)
62
+ report.indexSnapshots = input.indexSnapshots;
58
63
  const modelCalls = input.liveModelCalls ?? (input.evalCache ? input.evalCache.answerMisses + input.evalCache.judgeMisses : 0);
59
64
  report.liveProviderCalls += modelCalls;
60
65
  if (!input.evalCache && modelCalls > 0) {
@@ -77,6 +82,8 @@ function formatCacheReport(report) {
77
82
  part('evidence', report.evidence),
78
83
  part('answer', report.answer),
79
84
  part('judge', report.judge),
85
+ part('judge memo', report.judgeMemo),
86
+ report.indexSnapshots ? `index snapshots restored ${report.indexSnapshots.restored}/${report.indexSnapshots.restored + report.indexSnapshots.built + report.indexSnapshots.coldBusy} (${(report.indexSnapshots.savedIndexMs / 1000).toFixed(0)} s of index time not spent)` : null,
80
87
  report.uncached?.model ? `answer/judge uncached ${report.uncached.model}` : null,
81
88
  ].filter(Boolean);
82
89
  const probes = report.healthProbeCalls > 0 ? ` (+${report.healthProbeCalls} health probes, never cached)` : '';
@@ -22,7 +22,7 @@ exports.compileContext = compileContext;
22
22
  * header stubs (they carry session dates), recency-ordered like N22.
23
23
  * 3. Fill remaining budget by rank; no minimum raw count (tables win ties).
24
24
  */
25
- const gpt_tokenizer_1 = require("gpt-tokenizer");
25
+ const textTokens_1 = require("./textTokens");
26
26
  const canonicalSegment = (notePath) => notePath.replace(/^Sessions\//, '').replace(/-occ\d+\.md$/, '.md').replace(/\.md$/, '');
27
27
  exports.canonicalSegment = canonicalSegment;
28
28
  const isHeaderOnly = (text) => !text.includes('### Turn') && text.trim().length < 250;
@@ -80,7 +80,7 @@ function compileContext(args) {
80
80
  return `- ${f.value} ${f.unit ?? ''} ${f.object ?? ''}${f.event ? ` — ${f.event}` : ''}${co}`.replace(/\s+/g, ' ').trim();
81
81
  });
82
82
  const block = `Numeric facts:\n${lines.join('\n')}\nCite only values present here; otherwise use excerpts.`;
83
- const blockTokens = (0, gpt_tokenizer_1.encode)(block).length;
83
+ const blockTokens = (0, textTokens_1.countTextTokens)(block);
84
84
  if (blockTokens <= remaining) {
85
85
  excerpts.push({ text: block, date: null });
86
86
  tokens += blockTokens;
@@ -92,7 +92,7 @@ function compileContext(args) {
92
92
  if (chronScored.length) {
93
93
  const lines = chronScored.map(({ row: e }) => `- ${e.when ?? ''}: ${e.event ?? ''}`.replace(/\s+/g, ' ').trim());
94
94
  const block = `Events:\n${lines.join('\n')}\nCite only events present here; otherwise use excerpts.`;
95
- const blockTokens = (0, gpt_tokenizer_1.encode)(block).length;
95
+ const blockTokens = (0, textTokens_1.countTextTokens)(block);
96
96
  if (blockTokens <= remaining) {
97
97
  excerpts.push({ text: block, date: null });
98
98
  tokens += blockTokens;
@@ -104,7 +104,7 @@ function compileContext(args) {
104
104
  if (prefScored.length) {
105
105
  const lines = prefScored.map(({ row: p }) => `- ${p.statement ?? ''}`.replace(/\s+/g, ' ').trim());
106
106
  const block = `Standing preferences:\n${lines.join('\n')}\nApply matching entries; otherwise use excerpts.`;
107
- const blockTokens = (0, gpt_tokenizer_1.encode)(block).length;
107
+ const blockTokens = (0, textTokens_1.countTextTokens)(block);
108
108
  if (blockTokens <= remaining) {
109
109
  excerpts.push({ text: block, date: null });
110
110
  tokens += blockTokens;
@@ -129,7 +129,7 @@ function compileContext(args) {
129
129
  for (const raw of raws) {
130
130
  const dateMark = raw.date ? `(${compactDate(raw.date)}) ` : '';
131
131
  const candidate = `${dateMark}${raw.text}`;
132
- const candidateTokens = (0, gpt_tokenizer_1.encode)(candidate).length;
132
+ const candidateTokens = (0, textTokens_1.countTextTokens)(candidate);
133
133
  if (tokens + candidateTokens > maxContextTokens)
134
134
  break;
135
135
  tokens += candidateTokens;
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.WorkerPool = exports.KeyPool = exports.EvalCache = exports.CACHE_SCHEMA_VERSION = void 0;
6
+ exports.WorkerPool = exports.KeyPool = exports.EvalCache = exports.JUDGE_MEMO_SCOPE = exports.CACHE_SCHEMA_VERSION = void 0;
7
7
  exports.summarizeQaTokenUsage = summarizeQaTokenUsage;
8
8
  exports.runWithKeyPoolRetries = runWithKeyPoolRetries;
9
9
  exports.parseRetryAfterMs = parseRetryAfterMs;
@@ -14,7 +14,7 @@ exports.resolveJudgeKeys = resolveJudgeKeys;
14
14
  const node_fs_1 = __importDefault(require("node:fs"));
15
15
  const node_path_1 = __importDefault(require("node:path"));
16
16
  const node_crypto_1 = __importDefault(require("node:crypto"));
17
- const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
17
+ const sqlite_1 = __importDefault(require("../db/sqlite"));
18
18
  /**
19
19
  * Content-addressed eval cache for LongMemEval QA runs.
20
20
  *
@@ -68,6 +68,8 @@ const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
68
68
  * recorded in the run manifest.
69
69
  */
70
70
  exports.CACHE_SCHEMA_VERSION = 'evalcache-v4';
71
+ /** Routing-free scope of the judge memo (see EvalCache.computeJudgeMemoKey). Bump with its key formula. */
72
+ exports.JUDGE_MEMO_SCOPE = 'judge-memo-v1';
71
73
  function summarizeQaTokenUsage(records) {
72
74
  return {
73
75
  live: {
@@ -94,12 +96,13 @@ class EvalCache {
94
96
  liveAnswerPromptTokens: 0, liveAnswerCompletionTokens: 0,
95
97
  liveJudgePromptTokens: 0, liveJudgeCompletionTokens: 0,
96
98
  cacheLookupTotalMs: 0, cacheLookupCount: 0,
99
+ judgeMemoHits: 0, judgeMemoMisses: 0,
97
100
  };
98
101
  lookupStmt;
99
102
  insertStmt;
100
103
  constructor(dbPath) {
101
104
  node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
102
- this.db = new better_sqlite3_1.default(dbPath);
105
+ this.db = new sqlite_1.default(dbPath);
103
106
  this.db.pragma('journal_mode = WAL');
104
107
  this.db.pragma('busy_timeout = 10000');
105
108
  this.db.exec(`
@@ -150,6 +153,46 @@ class EvalCache {
150
153
  .update(`${exports.CACHE_SCHEMA_VERSION}\0${providerNamespace}\0${cacheScope}\0${JSON.stringify(requestBody)}`)
151
154
  .digest('hex');
152
155
  }
156
+ /**
157
+ * Key of the judge memo: one judge verdict per (question id, judge endpoint and model, and the whole judge
158
+ * request: system prompt, the question, the gold answer, the model answer, temperature and token cap).
159
+ * No routing scope: a verdict is a function of what the judge was asked, so a later run with a new routing
160
+ * session asks nothing new. The READER's answer is never keyed this way: it keeps the scope, so independent
161
+ * draws stay independent (only an identical answer text earns an identical verdict).
162
+ */
163
+ static computeJudgeMemoKey(providerNamespace, questionId, requestBody) {
164
+ return node_crypto_1.default.createHash('sha256')
165
+ .update(`${exports.CACHE_SCHEMA_VERSION}\0${providerNamespace}\0${exports.JUDGE_MEMO_SCOPE}\0${questionId}\0${JSON.stringify(requestBody)}`)
166
+ .digest('hex');
167
+ }
168
+ /** Look up a memoized judge verdict. A hit counts as a judge hit (and its avoided tokens); a miss is only counted as a memo miss. */
169
+ getJudgeMemo(memoKey) {
170
+ const row = this.lookupStmt.get(memoKey);
171
+ if (!row) {
172
+ this.stats.judgeMemoMisses++;
173
+ return null;
174
+ }
175
+ this.stats.judgeMemoHits++;
176
+ this.stats.judgeHits++;
177
+ this.stats.avoidedJudgePromptTokens += row.prompt_tokens;
178
+ this.stats.avoidedJudgeCompletionTokens += row.completion_tokens;
179
+ return {
180
+ rawContent: row.raw_content,
181
+ modelEcho: row.model_echo,
182
+ finishReason: row.finish_reason,
183
+ promptTokens: row.prompt_tokens,
184
+ completionTokens: row.completion_tokens,
185
+ latencyMs: row.latency_ms,
186
+ providerNamespace: row.provider_namespace,
187
+ };
188
+ }
189
+ /**
190
+ * Record a live judge verdict in the memo. Callers pass only verdicts that parsed: a response the parser
191
+ * refuses is retried by the next run, not replayed. Not a live call: put() already counted it.
192
+ */
193
+ putJudgeMemo(entry) {
194
+ return this.insertEntry(entry) > 0;
195
+ }
153
196
  /**
154
197
  * Look up a cached response. Returns null on miss.
155
198
  */