renor-engine 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +6 -6
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +2 -2
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +10 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/mcp/server.js +36 -6
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +310 -75
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +106 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +3 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +17 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +642 -127
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +93 -8
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/importing/index.js +0 -29
|
@@ -6,8 +6,33 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.VertexEmbeddingProvider = void 0;
|
|
7
7
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
8
8
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
9
|
+
const callPhases_1 = require("../callPhases");
|
|
9
10
|
const types_1 = require("./types");
|
|
10
|
-
|
|
11
|
+
/**
|
|
12
|
+
* How a token is kept fresh. Google's lives an hour. A call does not use one with a twelfth of its life left (5 of the 60
|
|
13
|
+
* minutes, the margin it always kept), and the token is renewed in the background when a sixth is left (10 minutes), so the
|
|
14
|
+
* calls keep finding one in hand and none of them waits for the exchange. Shares, not minutes, so a token of any lifetime is
|
|
15
|
+
* treated alike.
|
|
16
|
+
*/
|
|
17
|
+
const REQUEST_MARGIN_SHARE = 1 / 12;
|
|
18
|
+
const RENEW_AHEAD_SHARE = 1 / 6;
|
|
19
|
+
/**
|
|
20
|
+
* A token that lives less than this is not renewed in the background: the calls exchange for one when they need it. Google's
|
|
21
|
+
* lives an hour; a service that hands out tokens of seconds would otherwise be asked for one every few seconds for good.
|
|
22
|
+
*/
|
|
23
|
+
const MIN_RENEWED_LIFETIME_MS = 60_000;
|
|
24
|
+
/** A renewal that failed is tried again after this, for as long as the token in hand can still be used. */
|
|
25
|
+
const RENEW_RETRY_MS = 60_000;
|
|
26
|
+
/** The exchange is one small request, and every call that needs a token waits for it: a stalled one must not hold them all. */
|
|
27
|
+
const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000;
|
|
28
|
+
/** The longest wait setTimeout takes (2^31 - 1 ms); a longer one fires at once. */
|
|
29
|
+
const MAX_TIMER_MS = 2 ** 31 - 1;
|
|
30
|
+
/** The process's own, looked up at each use, so a global clock someone else has replaced is the one it reads. */
|
|
31
|
+
const processClock = {
|
|
32
|
+
now: () => Date.now(),
|
|
33
|
+
setTimeout: (callback, ms) => setTimeout(callback, ms),
|
|
34
|
+
clearTimeout: timer => clearTimeout(timer),
|
|
35
|
+
};
|
|
11
36
|
/** Quota (429) and transient (500/503) answers are retried: 8 attempts, waiting 10 s, 20 s ... up to 120 s between them. */
|
|
12
37
|
const DEFAULT_MAX_ATTEMPTS = 8;
|
|
13
38
|
const defaultRetryWaitMs = (attempt) => Math.min(120_000, 5_000 * 2 ** attempt) + Math.random() * 2_000;
|
|
@@ -40,6 +65,14 @@ function sleepUnlessAborted(ms, signal) {
|
|
|
40
65
|
function b64url(input) {
|
|
41
66
|
return Buffer.from(input).toString('base64url');
|
|
42
67
|
}
|
|
68
|
+
/** `work`, or the signal's reason as soon as it aborts: the caller stops waiting and `work` goes on for the others. */
|
|
69
|
+
function untilAborted(work, signal) {
|
|
70
|
+
return new Promise((resolve, reject) => {
|
|
71
|
+
const onAbort = () => reject(signal.reason);
|
|
72
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
73
|
+
work.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
|
|
74
|
+
});
|
|
75
|
+
}
|
|
43
76
|
class VertexEmbeddingProvider {
|
|
44
77
|
dimensions;
|
|
45
78
|
kind = 'vertex';
|
|
@@ -53,8 +86,16 @@ class VertexEmbeddingProvider {
|
|
|
53
86
|
base;
|
|
54
87
|
maxAttempts;
|
|
55
88
|
retryWaitMs;
|
|
89
|
+
/** Where its requests go: the global `fetch`, looked up at each call, unless the caller gave it another (see `request`). */
|
|
90
|
+
request;
|
|
91
|
+
tokenClock;
|
|
56
92
|
accessToken = null;
|
|
57
|
-
|
|
93
|
+
/** Epoch ms until which a call may use the token in hand (REQUEST_MARGIN_SHARE), and when it is renewed in the background. */
|
|
94
|
+
tokenUsableUntil = 0;
|
|
95
|
+
tokenRenewAt = 0;
|
|
96
|
+
/** The exchange in flight: a caller that needs a token while one is being fetched waits for it instead of making another. */
|
|
97
|
+
exchange = null;
|
|
98
|
+
renewTimer;
|
|
58
99
|
usage;
|
|
59
100
|
constructor(options) {
|
|
60
101
|
const keyFile = options?.keyFile
|
|
@@ -94,13 +135,60 @@ class VertexEmbeddingProvider {
|
|
|
94
135
|
this.base = endpointBase(options?.baseUrl, this.location);
|
|
95
136
|
this.maxAttempts = options?.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
96
137
|
this.retryWaitMs = options?.retry?.waitMs ?? defaultRetryWaitMs;
|
|
138
|
+
this.request = options?.request ?? ((input, init) => globalThis.fetch(input, init));
|
|
139
|
+
this.tokenClock = options?.tokenClock ?? processClock;
|
|
97
140
|
this.usage = { requests: 0, inputTokens: 0, maxInputTokens };
|
|
98
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* The token a call uses: the one in hand while it is usable, otherwise the exchange (one at a time, shared by every caller
|
|
144
|
+
* waiting for it). A caller that gives up (its signal aborts) stops waiting; the exchange goes on and the next call finds
|
|
145
|
+
* its token. The background renewal (armRenewal) keeps this from happening on a call at all while the provider is in use.
|
|
146
|
+
*/
|
|
147
|
+
getAccessToken(signal) {
|
|
148
|
+
if (this.accessToken && this.tokenClock.now() < this.tokenUsableUntil)
|
|
149
|
+
return Promise.resolve(this.accessToken);
|
|
150
|
+
if (signal?.aborted)
|
|
151
|
+
return Promise.reject(signal.reason); // as before: a call already stopped starts no exchange
|
|
152
|
+
return signal ? untilAborted(this.fetchToken(), signal) : this.fetchToken();
|
|
153
|
+
}
|
|
154
|
+
fetchToken() {
|
|
155
|
+
this.exchange ??= this.exchangeToken().finally(() => { this.exchange = null; });
|
|
156
|
+
return this.exchange;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* The token is renewed at RENEW_AHEAD_SHARE of its life left, whether or not a call is waiting for it, and the renewal
|
|
160
|
+
* arms the next one. A timer that cannot wait that long (setTimeout's limit) wakes early and waits again. The timer is
|
|
161
|
+
* unref'd: it never keeps a process alive. A provider that is no longer used goes on renewing while its process lives,
|
|
162
|
+
* one small request about every 50 minutes.
|
|
163
|
+
*/
|
|
164
|
+
armRenewal(at, lifetimeMs) {
|
|
165
|
+
this.tokenClock.clearTimeout(this.renewTimer);
|
|
166
|
+
if (!Number.isFinite(at) || !(lifetimeMs >= MIN_RENEWED_LIFETIME_MS))
|
|
167
|
+
return;
|
|
168
|
+
this.tokenRenewAt = at;
|
|
169
|
+
this.waitToRenew(at - this.tokenClock.now());
|
|
170
|
+
}
|
|
171
|
+
waitToRenew(ms) {
|
|
172
|
+
this.renewTimer = this.tokenClock.setTimeout(() => void this.renew(), Math.min(Math.max(0, ms), MAX_TIMER_MS));
|
|
173
|
+
this.renewTimer.unref();
|
|
174
|
+
}
|
|
175
|
+
async renew() {
|
|
176
|
+
const early = this.tokenRenewAt - this.tokenClock.now();
|
|
177
|
+
if (early > 0)
|
|
178
|
+
return this.waitToRenew(early);
|
|
179
|
+
try {
|
|
180
|
+
await this.fetchToken();
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// A call exchanges for a token when it must, and reports why if it cannot. Meanwhile, try again while the token in
|
|
184
|
+
// hand can still be used, so a service that was away for a moment is not found out by a call.
|
|
185
|
+
if (this.tokenClock.now() < this.tokenUsableUntil)
|
|
186
|
+
this.waitToRenew(RENEW_RETRY_MS);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
99
189
|
/** OAuth2 JWT-bearer grant: SA key -> short-lived access token. */
|
|
100
|
-
async
|
|
101
|
-
|
|
102
|
-
return this.accessToken;
|
|
103
|
-
const now = Math.floor(Date.now() / 1000);
|
|
190
|
+
async exchangeToken() {
|
|
191
|
+
const now = Math.floor(Date.now() / 1000); // the assertion is dated by the real clock, which is what Google reads
|
|
104
192
|
const header = b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
|
|
105
193
|
const claims = b64url(JSON.stringify({
|
|
106
194
|
iss: this.key.client_email,
|
|
@@ -112,20 +200,24 @@ class VertexEmbeddingProvider {
|
|
|
112
200
|
const signature = b64url(node_crypto_1.default.createSign('RSA-SHA256')
|
|
113
201
|
.update(`${header}.${claims}`)
|
|
114
202
|
.sign(this.key.private_key));
|
|
115
|
-
const response = await
|
|
203
|
+
const response = await this.request(this.key.token_uri, {
|
|
116
204
|
method: 'POST',
|
|
117
205
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
118
206
|
body: new URLSearchParams({
|
|
119
207
|
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
|
120
208
|
assertion: `${header}.${claims}.${signature}`,
|
|
121
209
|
}).toString(),
|
|
122
|
-
signal,
|
|
210
|
+
// Not a caller's signal: the exchange is shared, and one caller giving up must not end it for the others.
|
|
211
|
+
signal: AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS),
|
|
123
212
|
});
|
|
124
213
|
if (!response.ok)
|
|
125
214
|
throw new Error(`Vertex token exchange failed: ${response.status}`);
|
|
126
215
|
const data = await response.json();
|
|
216
|
+
const lifetimeMs = data.expires_in * 1000;
|
|
217
|
+
const issuedAt = this.tokenClock.now();
|
|
127
218
|
this.accessToken = data.access_token;
|
|
128
|
-
this.
|
|
219
|
+
this.tokenUsableUntil = issuedAt + lifetimeMs * (1 - REQUEST_MARGIN_SHARE);
|
|
220
|
+
this.armRenewal(issuedAt + lifetimeMs * (1 - RENEW_AHEAD_SHARE), lifetimeMs);
|
|
129
221
|
return this.accessToken;
|
|
130
222
|
}
|
|
131
223
|
/**
|
|
@@ -134,7 +226,9 @@ class VertexEmbeddingProvider {
|
|
|
134
226
|
* deadline. Without one the retries run their whole course.
|
|
135
227
|
*/
|
|
136
228
|
async predict(contents, call) {
|
|
229
|
+
const waitedForToken = performance.now();
|
|
137
230
|
const token = await this.getAccessToken(call.signal);
|
|
231
|
+
(0, callPhases_1.recordPhase)('token', performance.now() - waitedForToken);
|
|
138
232
|
const url = `${this.base}/v1/projects/${this.project}/locations/${this.location}/publishers/google/models/${this.model}:predict`;
|
|
139
233
|
const body = JSON.stringify({
|
|
140
234
|
instances: contents.map(({ text, taskType }) => ({
|
|
@@ -149,8 +243,15 @@ class VertexEmbeddingProvider {
|
|
|
149
243
|
const maxAttempts = this.maxAttempts;
|
|
150
244
|
let attempt = 0;
|
|
151
245
|
let response;
|
|
246
|
+
// The request itself, as the log's `call` phase has it: from sending to the last byte read, less the time spent opening a
|
|
247
|
+
// connection for it (which the pool reports as `connect`).
|
|
248
|
+
let sentAt = 0;
|
|
249
|
+
let connectedBefore = 0;
|
|
250
|
+
const spent = () => performance.now() - sentAt - ((0, callPhases_1.recordedPhase)('connect') - connectedBefore);
|
|
152
251
|
for (;;) {
|
|
153
|
-
|
|
252
|
+
sentAt = performance.now();
|
|
253
|
+
connectedBefore = (0, callPhases_1.recordedPhase)('connect');
|
|
254
|
+
response = await this.request(url, {
|
|
154
255
|
method: 'POST',
|
|
155
256
|
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
|
156
257
|
body,
|
|
@@ -158,6 +259,7 @@ class VertexEmbeddingProvider {
|
|
|
158
259
|
});
|
|
159
260
|
if (response.ok || ![429, 500, 503].includes(response.status))
|
|
160
261
|
break;
|
|
262
|
+
(0, callPhases_1.recordPhase)('call', spent());
|
|
161
263
|
attempt += 1;
|
|
162
264
|
if (attempt >= maxAttempts) {
|
|
163
265
|
throw new types_1.EmbeddingUnavailableError(`Vertex predict failed after ${attempt} attempts (HTTP ${response.status})`);
|
|
@@ -171,10 +273,12 @@ class VertexEmbeddingProvider {
|
|
|
171
273
|
}
|
|
172
274
|
if (!response.ok) {
|
|
173
275
|
const bodyText = await response.text();
|
|
276
|
+
(0, callPhases_1.recordPhase)('call', spent());
|
|
174
277
|
// The status travels with the error (the message is unchanged): a 400 names an input the service refuses.
|
|
175
278
|
throw Object.assign(new Error(`Vertex predict failed (${response.status}): ${bodyText.slice(0, 400)}`), { status: response.status });
|
|
176
279
|
}
|
|
177
280
|
const data = await response.json();
|
|
281
|
+
(0, callPhases_1.recordPhase)('call', spent());
|
|
178
282
|
const predictions = data.predictions ?? [];
|
|
179
283
|
if (predictions.length !== contents.length
|
|
180
284
|
|| predictions.some(p => !Array.isArray(p.embeddings?.values) || p.embeddings.values.length !== this.dimensions)) {
|
|
@@ -196,9 +300,15 @@ class VertexEmbeddingProvider {
|
|
|
196
300
|
signal: options?.signal,
|
|
197
301
|
};
|
|
198
302
|
const out = [];
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
303
|
+
const started = performance.now();
|
|
304
|
+
try {
|
|
305
|
+
for (let i = 0; i < texts.length; i += this.maxInstancesPerRequest) {
|
|
306
|
+
const batch = texts.slice(i, i + this.maxInstancesPerRequest).map(text => ({ text, taskType }));
|
|
307
|
+
out.push(...await this.predict(batch, call));
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
finally {
|
|
311
|
+
(0, callPhases_1.recordPhase)('embed', performance.now() - started);
|
|
202
312
|
}
|
|
203
313
|
return out;
|
|
204
314
|
}
|
package/dist/eval/cacheReport.js
CHANGED
|
@@ -52,9 +52,14 @@ function buildCacheReport(input) {
|
|
|
52
52
|
const { answerHits, answerMisses, judgeHits, judgeMisses } = input.evalCache;
|
|
53
53
|
report.answer = rate({ hits: answerHits, misses: answerMisses });
|
|
54
54
|
report.judge = rate({ hits: judgeHits, misses: judgeMisses });
|
|
55
|
+
if ((input.evalCache.judgeMemoHits ?? 0) + (input.evalCache.judgeMemoMisses ?? 0) > 0) {
|
|
56
|
+
report.judgeMemo = rate({ hits: input.evalCache.judgeMemoHits ?? 0, misses: input.evalCache.judgeMemoMisses ?? 0 });
|
|
57
|
+
}
|
|
55
58
|
if (!input.pinnedScope && answerHits + answerMisses + judgeHits + judgeMisses > 0)
|
|
56
59
|
report.repeatRunBlockers.push(exports.UNPINNED_SCOPE_BLOCKER);
|
|
57
60
|
}
|
|
61
|
+
if (input.indexSnapshots)
|
|
62
|
+
report.indexSnapshots = input.indexSnapshots;
|
|
58
63
|
const modelCalls = input.liveModelCalls ?? (input.evalCache ? input.evalCache.answerMisses + input.evalCache.judgeMisses : 0);
|
|
59
64
|
report.liveProviderCalls += modelCalls;
|
|
60
65
|
if (!input.evalCache && modelCalls > 0) {
|
|
@@ -77,6 +82,8 @@ function formatCacheReport(report) {
|
|
|
77
82
|
part('evidence', report.evidence),
|
|
78
83
|
part('answer', report.answer),
|
|
79
84
|
part('judge', report.judge),
|
|
85
|
+
part('judge memo', report.judgeMemo),
|
|
86
|
+
report.indexSnapshots ? `index snapshots restored ${report.indexSnapshots.restored}/${report.indexSnapshots.restored + report.indexSnapshots.built + report.indexSnapshots.coldBusy} (${(report.indexSnapshots.savedIndexMs / 1000).toFixed(0)} s of index time not spent)` : null,
|
|
80
87
|
report.uncached?.model ? `answer/judge uncached ${report.uncached.model}` : null,
|
|
81
88
|
].filter(Boolean);
|
|
82
89
|
const probes = report.healthProbeCalls > 0 ? ` (+${report.healthProbeCalls} health probes, never cached)` : '';
|
|
@@ -22,7 +22,7 @@ exports.compileContext = compileContext;
|
|
|
22
22
|
* header stubs (they carry session dates), recency-ordered like N22.
|
|
23
23
|
* 3. Fill remaining budget by rank; no minimum raw count (tables win ties).
|
|
24
24
|
*/
|
|
25
|
-
const
|
|
25
|
+
const textTokens_1 = require("./textTokens");
|
|
26
26
|
const canonicalSegment = (notePath) => notePath.replace(/^Sessions\//, '').replace(/-occ\d+\.md$/, '.md').replace(/\.md$/, '');
|
|
27
27
|
exports.canonicalSegment = canonicalSegment;
|
|
28
28
|
const isHeaderOnly = (text) => !text.includes('### Turn') && text.trim().length < 250;
|
|
@@ -80,7 +80,7 @@ function compileContext(args) {
|
|
|
80
80
|
return `- ${f.value} ${f.unit ?? ''} ${f.object ?? ''}${f.event ? ` — ${f.event}` : ''}${co}`.replace(/\s+/g, ' ').trim();
|
|
81
81
|
});
|
|
82
82
|
const block = `Numeric facts:\n${lines.join('\n')}\nCite only values present here; otherwise use excerpts.`;
|
|
83
|
-
const blockTokens = (0,
|
|
83
|
+
const blockTokens = (0, textTokens_1.countTextTokens)(block);
|
|
84
84
|
if (blockTokens <= remaining) {
|
|
85
85
|
excerpts.push({ text: block, date: null });
|
|
86
86
|
tokens += blockTokens;
|
|
@@ -92,7 +92,7 @@ function compileContext(args) {
|
|
|
92
92
|
if (chronScored.length) {
|
|
93
93
|
const lines = chronScored.map(({ row: e }) => `- ${e.when ?? ''}: ${e.event ?? ''}`.replace(/\s+/g, ' ').trim());
|
|
94
94
|
const block = `Events:\n${lines.join('\n')}\nCite only events present here; otherwise use excerpts.`;
|
|
95
|
-
const blockTokens = (0,
|
|
95
|
+
const blockTokens = (0, textTokens_1.countTextTokens)(block);
|
|
96
96
|
if (blockTokens <= remaining) {
|
|
97
97
|
excerpts.push({ text: block, date: null });
|
|
98
98
|
tokens += blockTokens;
|
|
@@ -104,7 +104,7 @@ function compileContext(args) {
|
|
|
104
104
|
if (prefScored.length) {
|
|
105
105
|
const lines = prefScored.map(({ row: p }) => `- ${p.statement ?? ''}`.replace(/\s+/g, ' ').trim());
|
|
106
106
|
const block = `Standing preferences:\n${lines.join('\n')}\nApply matching entries; otherwise use excerpts.`;
|
|
107
|
-
const blockTokens = (0,
|
|
107
|
+
const blockTokens = (0, textTokens_1.countTextTokens)(block);
|
|
108
108
|
if (blockTokens <= remaining) {
|
|
109
109
|
excerpts.push({ text: block, date: null });
|
|
110
110
|
tokens += blockTokens;
|
|
@@ -129,7 +129,7 @@ function compileContext(args) {
|
|
|
129
129
|
for (const raw of raws) {
|
|
130
130
|
const dateMark = raw.date ? `(${compactDate(raw.date)}) ` : '';
|
|
131
131
|
const candidate = `${dateMark}${raw.text}`;
|
|
132
|
-
const candidateTokens = (0,
|
|
132
|
+
const candidateTokens = (0, textTokens_1.countTextTokens)(candidate);
|
|
133
133
|
if (tokens + candidateTokens > maxContextTokens)
|
|
134
134
|
break;
|
|
135
135
|
tokens += candidateTokens;
|
package/dist/eval/evalCache.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.WorkerPool = exports.KeyPool = exports.EvalCache = exports.CACHE_SCHEMA_VERSION = void 0;
|
|
6
|
+
exports.WorkerPool = exports.KeyPool = exports.EvalCache = exports.JUDGE_MEMO_SCOPE = exports.CACHE_SCHEMA_VERSION = void 0;
|
|
7
7
|
exports.summarizeQaTokenUsage = summarizeQaTokenUsage;
|
|
8
8
|
exports.runWithKeyPoolRetries = runWithKeyPoolRetries;
|
|
9
9
|
exports.parseRetryAfterMs = parseRetryAfterMs;
|
|
@@ -14,7 +14,7 @@ exports.resolveJudgeKeys = resolveJudgeKeys;
|
|
|
14
14
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
15
15
|
const node_path_1 = __importDefault(require("node:path"));
|
|
16
16
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
17
|
-
const
|
|
17
|
+
const sqlite_1 = __importDefault(require("../db/sqlite"));
|
|
18
18
|
/**
|
|
19
19
|
* Content-addressed eval cache for LongMemEval QA runs.
|
|
20
20
|
*
|
|
@@ -68,6 +68,8 @@ const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
|
68
68
|
* recorded in the run manifest.
|
|
69
69
|
*/
|
|
70
70
|
exports.CACHE_SCHEMA_VERSION = 'evalcache-v4';
|
|
71
|
+
/** Routing-free scope of the judge memo (see EvalCache.computeJudgeMemoKey). Bump with its key formula. */
|
|
72
|
+
exports.JUDGE_MEMO_SCOPE = 'judge-memo-v1';
|
|
71
73
|
function summarizeQaTokenUsage(records) {
|
|
72
74
|
return {
|
|
73
75
|
live: {
|
|
@@ -94,12 +96,13 @@ class EvalCache {
|
|
|
94
96
|
liveAnswerPromptTokens: 0, liveAnswerCompletionTokens: 0,
|
|
95
97
|
liveJudgePromptTokens: 0, liveJudgeCompletionTokens: 0,
|
|
96
98
|
cacheLookupTotalMs: 0, cacheLookupCount: 0,
|
|
99
|
+
judgeMemoHits: 0, judgeMemoMisses: 0,
|
|
97
100
|
};
|
|
98
101
|
lookupStmt;
|
|
99
102
|
insertStmt;
|
|
100
103
|
constructor(dbPath) {
|
|
101
104
|
node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
|
|
102
|
-
this.db = new
|
|
105
|
+
this.db = new sqlite_1.default(dbPath);
|
|
103
106
|
this.db.pragma('journal_mode = WAL');
|
|
104
107
|
this.db.pragma('busy_timeout = 10000');
|
|
105
108
|
this.db.exec(`
|
|
@@ -150,6 +153,46 @@ class EvalCache {
|
|
|
150
153
|
.update(`${exports.CACHE_SCHEMA_VERSION}\0${providerNamespace}\0${cacheScope}\0${JSON.stringify(requestBody)}`)
|
|
151
154
|
.digest('hex');
|
|
152
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Key of the judge memo: one judge verdict per (question id, judge endpoint and model, and the whole judge
|
|
158
|
+
* request: system prompt, the question, the gold answer, the model answer, temperature and token cap).
|
|
159
|
+
* No routing scope: a verdict is a function of what the judge was asked, so a later run with a new routing
|
|
160
|
+
* session asks nothing new. The READER's answer is never keyed this way: it keeps the scope, so independent
|
|
161
|
+
* draws stay independent (only an identical answer text earns an identical verdict).
|
|
162
|
+
*/
|
|
163
|
+
static computeJudgeMemoKey(providerNamespace, questionId, requestBody) {
|
|
164
|
+
return node_crypto_1.default.createHash('sha256')
|
|
165
|
+
.update(`${exports.CACHE_SCHEMA_VERSION}\0${providerNamespace}\0${exports.JUDGE_MEMO_SCOPE}\0${questionId}\0${JSON.stringify(requestBody)}`)
|
|
166
|
+
.digest('hex');
|
|
167
|
+
}
|
|
168
|
+
/** Look up a memoized judge verdict. A hit counts as a judge hit (and its avoided tokens); a miss is only counted as a memo miss. */
|
|
169
|
+
getJudgeMemo(memoKey) {
|
|
170
|
+
const row = this.lookupStmt.get(memoKey);
|
|
171
|
+
if (!row) {
|
|
172
|
+
this.stats.judgeMemoMisses++;
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
this.stats.judgeMemoHits++;
|
|
176
|
+
this.stats.judgeHits++;
|
|
177
|
+
this.stats.avoidedJudgePromptTokens += row.prompt_tokens;
|
|
178
|
+
this.stats.avoidedJudgeCompletionTokens += row.completion_tokens;
|
|
179
|
+
return {
|
|
180
|
+
rawContent: row.raw_content,
|
|
181
|
+
modelEcho: row.model_echo,
|
|
182
|
+
finishReason: row.finish_reason,
|
|
183
|
+
promptTokens: row.prompt_tokens,
|
|
184
|
+
completionTokens: row.completion_tokens,
|
|
185
|
+
latencyMs: row.latency_ms,
|
|
186
|
+
providerNamespace: row.provider_namespace,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Record a live judge verdict in the memo. Callers pass only verdicts that parsed: a response the parser
|
|
191
|
+
* refuses is retried by the next run, not replayed. Not a live call: put() already counted it.
|
|
192
|
+
*/
|
|
193
|
+
putJudgeMemo(entry) {
|
|
194
|
+
return this.insertEntry(entry) > 0;
|
|
195
|
+
}
|
|
153
196
|
/**
|
|
154
197
|
* Look up a cached response. Returns null on miss.
|
|
155
198
|
*/
|