total-recall-brain 3.19.2 → 3.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scaffold/.agent/skills/total-recall/modules/agents/agents.yml +51 -15
- package/src/cli/recall.mjs +66 -7
- package/src/core/config.mjs +7 -1
- package/src/core/runtime.mjs +162 -18
- package/src/core/search.mjs +19 -2
- package/src/core/throttled-fetch.mjs +17 -0
- package/src/server/index.mjs +54 -1
- package/src/server/routes/vault.mjs +13 -2
package/package.json
CHANGED
|
@@ -1,30 +1,66 @@
|
|
|
1
1
|
agents:
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
# Google Antigravity CLI. The binary is `agy` (NOT `antigravity`) and it runs
|
|
3
|
+
# on the AI Ultra plan. Preferred agent for bulk work.
|
|
4
|
+
# agy -p "prompt" --output-format json -> {status:"SUCCESS", response:"..."}
|
|
5
|
+
# Docs: https://antigravity.google/docs/cli/headless
|
|
6
|
+
- name: agy
|
|
7
|
+
binary: agy
|
|
8
|
+
flags: --output-format json -p
|
|
5
9
|
priority: 1
|
|
6
10
|
enabled: true
|
|
7
11
|
exec: flag
|
|
8
12
|
|
|
9
|
-
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
priority: 2
|
|
14
|
-
enabled: true
|
|
15
|
-
exec: flag
|
|
16
|
-
|
|
13
|
+
# `-p` MUST be present (non-interactive requires --print) and MUST be LAST:
|
|
14
|
+
# `--tools` is variadic (`<tools...>`) in the shipped build, so anything after
|
|
15
|
+
# it — including the appended prompt — is swallowed as a tool name. Proven
|
|
16
|
+
# both ways on claude v2.1.201.
|
|
17
17
|
- name: claude
|
|
18
18
|
binary: claude
|
|
19
|
-
flags: --output-format json --permission-mode bypassPermissions --setting-sources local --tools ""
|
|
20
|
-
priority:
|
|
19
|
+
flags: --output-format json --permission-mode bypassPermissions --setting-sources local --tools "" -p
|
|
20
|
+
priority: 2
|
|
21
21
|
enabled: true
|
|
22
22
|
exec: flag
|
|
23
|
+
|
|
24
|
+
# `-m` is REQUIRED: ~/.codex/config.toml may pin a model the account cannot
|
|
25
|
+
# use (seen: gpt-5.6-sol -> HTTP 400 "not supported when using Codex with a
|
|
26
|
+
# ChatGPT account"). Account-supported slugs: ~/.codex/models_cache.json.
|
|
27
|
+
# `--full-auto` is hidden from --help but still works as a deprecated alias;
|
|
28
|
+
# the explicit `--sandbox` form is OpenAI's current recommendation.
|
|
23
29
|
- name: codex
|
|
24
30
|
binary: codex
|
|
25
|
-
flags: --
|
|
26
|
-
priority:
|
|
31
|
+
flags: -m gpt-5.5 --sandbox workspace-write --json --skip-git-repo-check
|
|
32
|
+
priority: 3
|
|
27
33
|
enabled: true
|
|
28
34
|
exec: subcommand
|
|
35
|
+
|
|
36
|
+
# DISABLED — `antigravity` on PATH is total-recall/bin/antigravity.mjs, a thin
|
|
37
|
+
# wrapper that POSTs to generativelanguage.googleapis.com with GOOGLE_API_KEY,
|
|
38
|
+
# i.e. the METERED Gemini API, not the Ultra plan. Verified 2026-08-01: every
|
|
39
|
+
# call returned 429 "You exceeded your current quota". Use `agy` above.
|
|
40
|
+
- name: antigravity
|
|
41
|
+
binary: antigravity
|
|
42
|
+
flags: --sandbox=false --yolo -o json
|
|
43
|
+
priority: 8
|
|
44
|
+
enabled: false
|
|
45
|
+
exec: flag
|
|
46
|
+
|
|
47
|
+
# DISABLED — this user has no Gemini CLI. Do not enable because a `gemini`
|
|
48
|
+
# binary resolves on PATH; their Google surface is `agy`.
|
|
49
|
+
- name: gemini
|
|
50
|
+
binary: gemini
|
|
51
|
+
flags: --sandbox=false --yolo -o json
|
|
52
|
+
priority: 9
|
|
53
|
+
enabled: false
|
|
54
|
+
exec: flag
|
|
55
|
+
|
|
56
|
+
# DISABLED — not installed. A stale binary may still resolve at
|
|
57
|
+
# ~/.grok/bin/grok; PATH presence is not proof an agent works.
|
|
58
|
+
- name: grok
|
|
59
|
+
binary: grok
|
|
60
|
+
flags: --output-format plain --always-approve --permission-mode bypassPermissions
|
|
61
|
+
priority: 10
|
|
62
|
+
enabled: false
|
|
63
|
+
exec: flag
|
|
64
|
+
|
|
29
65
|
timeout: 300
|
|
30
66
|
max_retries: 2
|
package/src/cli/recall.mjs
CHANGED
|
@@ -128,6 +128,8 @@ export default async function recall(args) {
|
|
|
128
128
|
// Search all target brains and merge results
|
|
129
129
|
let allResults = [];
|
|
130
130
|
let isDegraded = false;
|
|
131
|
+
const searchErrors = [];
|
|
132
|
+
const degradedReasons = new Map();
|
|
131
133
|
|
|
132
134
|
for (const target of searchTargets) {
|
|
133
135
|
const vaultDir = path.join(target.brainDir, 'memory-vault');
|
|
@@ -162,14 +164,28 @@ export default async function recall(args) {
|
|
|
162
164
|
priority
|
|
163
165
|
});
|
|
164
166
|
}
|
|
165
|
-
if (results.degradedTextSearch)
|
|
167
|
+
if (results.degradedTextSearch) {
|
|
168
|
+
isDegraded = true;
|
|
169
|
+
degradedReasons.set(results.degradedReason || 'unknown', {
|
|
170
|
+
label: target.label,
|
|
171
|
+
detail: results.degradedDetail || '',
|
|
172
|
+
});
|
|
173
|
+
}
|
|
166
174
|
// Tag each result with its layer
|
|
167
175
|
for (const r of results) {
|
|
168
176
|
r._layer = target.label;
|
|
169
177
|
}
|
|
170
178
|
allResults = allResults.concat(results);
|
|
171
|
-
} catch {
|
|
172
|
-
//
|
|
179
|
+
} catch (err) {
|
|
180
|
+
// A brain that cannot be searched is skipped — but NEVER silently.
|
|
181
|
+
//
|
|
182
|
+
// This used to be a bare `catch {}`. A missing native binding for
|
|
183
|
+
// better-sqlite3 ("Could not locate the bindings file") made every
|
|
184
|
+
// semantic search throw, and the swallow turned a hard dependency
|
|
185
|
+
// crash into the message "No matching memory nodes found" — so a
|
|
186
|
+
// fully-populated vault looked empty and the real cause was invisible.
|
|
187
|
+
// Degrading to lexical-only results is fine; hiding why is not.
|
|
188
|
+
searchErrors.push({ label: target.label, message: String(err?.message || err) });
|
|
173
189
|
}
|
|
174
190
|
}
|
|
175
191
|
|
|
@@ -182,16 +198,59 @@ export default async function recall(args) {
|
|
|
182
198
|
return;
|
|
183
199
|
}
|
|
184
200
|
|
|
201
|
+
if (searchErrors.length > 0) {
|
|
202
|
+
// Surface the real cause. "No matches" and "search crashed" are very
|
|
203
|
+
// different problems and must never look the same.
|
|
204
|
+
console.log('');
|
|
205
|
+
for (const e of searchErrors) {
|
|
206
|
+
console.log(` ⚠️ ${e.label} brain search failed: ${e.message.split('\n')[0]}`);
|
|
207
|
+
}
|
|
208
|
+
if (/bindings file|\.node|NODE_MODULE_VERSION|better-sqlite3/i.test(searchErrors.map(e => e.message).join(' '))) {
|
|
209
|
+
console.log(' A native module is missing or was built for a different Node version.');
|
|
210
|
+
console.log(' Fix: pnpm rebuild better-sqlite3 (or npm rebuild better-sqlite3)');
|
|
211
|
+
console.log(' With pnpm, the package must also be allowed to run build scripts.');
|
|
212
|
+
}
|
|
213
|
+
console.log(' Results below (if any) are lexical-only and may be incomplete.');
|
|
214
|
+
}
|
|
215
|
+
|
|
185
216
|
if (allResults.length === 0) {
|
|
186
|
-
console.log(
|
|
217
|
+
console.log(
|
|
218
|
+
searchErrors.length > 0
|
|
219
|
+
? '\n 🔍 No results — search failed above, so this is NOT proof the vault is empty.'
|
|
220
|
+
: '\n 🔍 No matching memory nodes found.',
|
|
221
|
+
);
|
|
187
222
|
return;
|
|
188
223
|
}
|
|
189
224
|
if (isDegraded) {
|
|
190
|
-
|
|
191
|
-
|
|
225
|
+
// This block used to print ONE message for two unrelated failures:
|
|
226
|
+
// "Embedding API unreachable or unconfigured … Configure GOOGLE_API_KEY
|
|
227
|
+
// or OPENAI_API_KEY". When the real cause was an unbuilt index, that
|
|
228
|
+
// sent the reader to check credentials that were already working, while
|
|
229
|
+
// a brain with thousands of nodes quietly answered from keyword matching
|
|
230
|
+
// and returned results from the wrong project. Name the actual cause.
|
|
231
|
+
const empty = degradedReasons.get('empty_index');
|
|
232
|
+
const failed = degradedReasons.get('embedding_failed');
|
|
233
|
+
|
|
234
|
+
console.log('\n ⚠️ Vector search is OFF — these results are keyword-only and may miss obvious matches.');
|
|
235
|
+
if (empty) {
|
|
236
|
+
console.log(` Cause: the ${empty.label} brain has NO embeddings indexed. The API is fine; the index was never built.`);
|
|
237
|
+
console.log(' Fix: npx total-recall compile (embeds any unembedded nodes for this brain)');
|
|
238
|
+
}
|
|
239
|
+
if (failed) {
|
|
240
|
+
console.log(` Cause: embedding generation failed for the ${failed.label} brain — ${failed.detail}`);
|
|
241
|
+
console.log(' Fix: set OPENROUTER_API_KEY (preferred), or GOOGLE_API_KEY / OPENAI_API_KEY.');
|
|
242
|
+
}
|
|
243
|
+
if (!empty && !failed) {
|
|
244
|
+
console.log(' Cause: unknown — embeddings unavailable and no reason was reported.');
|
|
245
|
+
}
|
|
192
246
|
}
|
|
193
247
|
const layerSuffix = searchTargets.length > 1 ? ' (merged)' : ` (${searchTargets[0].label})`;
|
|
194
|
-
|
|
248
|
+
// Never call it "Semantic search results" when semantic search did not run.
|
|
249
|
+
// The old header said exactly that regardless, so degraded keyword output
|
|
250
|
+
// was indistinguishable from real vector results — same header, same
|
|
251
|
+
// percentage scores. If the label doesn't change, nobody can tell.
|
|
252
|
+
const modeLabel = isDegraded ? '🔤 KEYWORD-ONLY results' : '🔍 Semantic search results';
|
|
253
|
+
console.log(`\n ${modeLabel} for "${query}"${layerSuffix}:\n`);
|
|
195
254
|
for (let i = 0; i < allResults.length; i++) {
|
|
196
255
|
const match = allResults[i];
|
|
197
256
|
const rank = i + 1;
|
package/src/core/config.mjs
CHANGED
|
@@ -285,7 +285,13 @@ export function detectProjectBrain(startDir = process.cwd()) {
|
|
|
285
285
|
if (fs.existsSync(path.join(candidate, 'SKILL.md'))) {
|
|
286
286
|
return {
|
|
287
287
|
agentDir: path.join(dir, '.agent'),
|
|
288
|
-
|
|
288
|
+
// The project brain lives at the detected candidate — NEVER alias it to
|
|
289
|
+
// globalBrainDir. This returned `globalBrainDir` while the sibling
|
|
290
|
+
// implementation in src/cli/agent-dir.mjs returned `candidate`, so the
|
|
291
|
+
// two disagreed about where the project brain is: anything routed
|
|
292
|
+
// through this function silently operated on the GLOBAL vault while
|
|
293
|
+
// reporting itself as the 'project' layer.
|
|
294
|
+
brainDir: candidate,
|
|
289
295
|
projectRoot: dir,
|
|
290
296
|
};
|
|
291
297
|
}
|
package/src/core/runtime.mjs
CHANGED
|
@@ -47,11 +47,21 @@ export function findBinaryInPath(binaryName) {
|
|
|
47
47
|
|
|
48
48
|
// ─── Default agent registry (used when agents.yml doesn't exist) ─────────────
|
|
49
49
|
|
|
50
|
+
// NOTE ON FLAG ORDER AND MODEL PINS (verified by live probe 2026-08-01):
|
|
51
|
+
// agy — the REAL Google Antigravity CLI (AI Ultra). The binary named
|
|
52
|
+
// `antigravity` is a metered Gemini API wrapper, NOT this. Disabled
|
|
53
|
+
// below; enabling it bills an exhausted API budget (429s).
|
|
54
|
+
// claude — `-p` is required for non-interactive AND must be LAST, because
|
|
55
|
+
// `--tools` is variadic and otherwise swallows the prompt.
|
|
56
|
+
// codex — `-m` must be pinned; a config.toml model the account cannot use
|
|
57
|
+
// fails with HTTP 400 at exit code 0.
|
|
50
58
|
const DEFAULT_AGENTS = [
|
|
51
|
-
{ name: '
|
|
52
|
-
{ name: '
|
|
53
|
-
{ name: '
|
|
54
|
-
{ name: '
|
|
59
|
+
{ name: 'agy', binary: 'agy', flags: '--output-format json -p', priority: 1, enabled: true, exec: 'flag' },
|
|
60
|
+
{ name: 'claude', binary: 'claude', flags: '--output-format json --permission-mode bypassPermissions --setting-sources local --tools "" -p', priority: 2, enabled: true, exec: 'flag' },
|
|
61
|
+
{ name: 'codex', binary: 'codex', flags: '-m gpt-5.5 --sandbox workspace-write --json --skip-git-repo-check', priority: 3, enabled: true, exec: 'subcommand' },
|
|
62
|
+
{ name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 8, enabled: false, exec: 'flag' },
|
|
63
|
+
{ name: 'gemini', binary: 'gemini', flags: '--sandbox=false --yolo -o json', priority: 9, enabled: false, exec: 'flag' },
|
|
64
|
+
{ name: 'grok', binary: 'grok', flags: '--output-format plain --always-approve --permission-mode bypassPermissions', priority: 10, enabled: false, exec: 'flag' },
|
|
55
65
|
];
|
|
56
66
|
|
|
57
67
|
/**
|
|
@@ -299,29 +309,65 @@ export async function callLocalRuntime(prompt, system, config) {
|
|
|
299
309
|
}
|
|
300
310
|
} catch {}
|
|
301
311
|
|
|
302
|
-
// Try agents in priority order
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
312
|
+
// Try agents in priority order, never re-trying one that already failed.
|
|
313
|
+
//
|
|
314
|
+
// The old filter excluded only `lastError.agentName` — the MOST RECENT
|
|
315
|
+
// failure — so a chain of three could hand work back to an agent that had
|
|
316
|
+
// already failed while a healthy one further down was never reached. Track
|
|
317
|
+
// every failure instead.
|
|
318
|
+
//
|
|
319
|
+
// The iteration cap is `agents.length + maxRetries` so that each enabled
|
|
320
|
+
// agent gets a turn even when the retry budget is smaller than the chain;
|
|
321
|
+
// the loop exits as soon as the untried pool is empty.
|
|
322
|
+
const failedAgents = new Set();
|
|
323
|
+
const maxAttempts = agents.length + maxRetries;
|
|
324
|
+
|
|
325
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
326
|
+
const remaining = agents.filter(a => !failedAgents.has(a.name));
|
|
327
|
+
const agent = remaining.length > 0
|
|
328
|
+
? resolveAgent({ ...config, agents: remaining })
|
|
329
|
+
: null;
|
|
308
330
|
|
|
309
331
|
if (!agent) {
|
|
310
|
-
|
|
332
|
+
if (failedAgents.size > 0) {
|
|
333
|
+
throw new Error(
|
|
334
|
+
`All CLI agents failed (tried: ${[...failedAgents].join(', ')}). ` +
|
|
335
|
+
`Last error: ${lastError?.error || 'unknown'}`,
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
throw new Error(
|
|
339
|
+
'No CLI agents available. Install agy (Google Antigravity), claude, or codex.',
|
|
340
|
+
);
|
|
311
341
|
}
|
|
312
342
|
|
|
343
|
+
// The prompt goes LAST, after every flag. Pushing it first (the old
|
|
344
|
+
// behaviour) broke the registry's own flag sets: it emitted a duplicate
|
|
345
|
+
// `-p` for agents whose flags already carry one, and it defeats the
|
|
346
|
+
// ordering that keeps a variadic flag (claude's `--tools <tools...>`) from
|
|
347
|
+
// swallowing the prompt as one of its values.
|
|
313
348
|
const args = [];
|
|
314
349
|
if (agent.exec === 'subcommand') {
|
|
315
|
-
args.push('exec'
|
|
316
|
-
} else {
|
|
317
|
-
args.push('-p', fullPrompt);
|
|
350
|
+
args.push('exec');
|
|
318
351
|
}
|
|
319
352
|
if (agent.flags) {
|
|
320
|
-
|
|
353
|
+
// spawnSync does not go through a shell, so quotes in the registry are
|
|
354
|
+
// literal characters: `--tools ""` would pass a two-character string
|
|
355
|
+
// rather than an empty one. Strip matched surrounding quotes per token.
|
|
356
|
+
args.push(
|
|
357
|
+
...agent.flags
|
|
358
|
+
.split(/\s+/)
|
|
359
|
+
.filter(Boolean)
|
|
360
|
+
.map(tok => tok.replace(/^(['"])(.*)\1$/, '$2')),
|
|
361
|
+
);
|
|
321
362
|
}
|
|
322
363
|
if (agent.model) {
|
|
323
364
|
args.push('-m', agent.model);
|
|
324
365
|
}
|
|
366
|
+
// `exec: flag` agents need an explicit `-p` only if their flags omit it.
|
|
367
|
+
if (agent.exec !== 'subcommand' && !args.includes('-p') && !args.includes('--print')) {
|
|
368
|
+
args.push('-p');
|
|
369
|
+
}
|
|
370
|
+
args.push(fullPrompt);
|
|
325
371
|
|
|
326
372
|
const cmd = `${agent.binaryPath} ${args.map(a => a.includes(' ') || a.includes('\n') ? `"${a.replace(/"/g, '\\"')}"` : a).join(' ')}`;
|
|
327
373
|
|
|
@@ -373,11 +419,27 @@ export async function callLocalRuntime(prompt, system, config) {
|
|
|
373
419
|
cwd: process.cwd(),
|
|
374
420
|
});
|
|
375
421
|
|
|
376
|
-
|
|
422
|
+
const softFailure = detectAgentFailure(result.stdout || '');
|
|
423
|
+
|
|
424
|
+
if (result.status === 0 && !softFailure) {
|
|
377
425
|
const output = result.stdout?.trim() || '';
|
|
378
426
|
return parseAgentOutput(output);
|
|
379
427
|
}
|
|
380
428
|
|
|
429
|
+
if (softFailure) {
|
|
430
|
+
// Exit code 0 but the payload says the run failed. Auth and model-config
|
|
431
|
+
// failures are not transient, so record and move to the next agent
|
|
432
|
+
// instead of burning the remaining retries on the same broken agent.
|
|
433
|
+
logger.error({
|
|
434
|
+
subsystem: 'runtime',
|
|
435
|
+
message: `${agent.name} reported failure despite exit 0`,
|
|
436
|
+
error: softFailure.slice(0, 500),
|
|
437
|
+
});
|
|
438
|
+
lastError = { agentName: agent.name, error: softFailure.slice(0, 500) };
|
|
439
|
+
failedAgents.add(agent.name);
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
|
|
381
443
|
const err = result.stderr || result.stdout || 'Unknown error';
|
|
382
444
|
logger.error({
|
|
383
445
|
subsystem: 'runtime',
|
|
@@ -385,9 +447,13 @@ export async function callLocalRuntime(prompt, system, config) {
|
|
|
385
447
|
error: err.slice(0, 500),
|
|
386
448
|
});
|
|
387
449
|
lastError = { agentName: agent.name, error: err.slice(0, 500) };
|
|
450
|
+
failedAgents.add(agent.name);
|
|
388
451
|
}
|
|
389
452
|
|
|
390
|
-
throw new Error(
|
|
453
|
+
throw new Error(
|
|
454
|
+
`All CLI agents failed (tried: ${[...failedAgents].join(', ') || 'none'}). ` +
|
|
455
|
+
`Last error: ${lastError?.error || 'unknown'}`,
|
|
456
|
+
);
|
|
391
457
|
}
|
|
392
458
|
|
|
393
459
|
/**
|
|
@@ -406,6 +472,57 @@ export async function callLocalRuntimeRaw(messages, config, _tools = undefined)
|
|
|
406
472
|
/**
|
|
407
473
|
* Parse JSON or text output from a CLI agent into plain text.
|
|
408
474
|
*/
|
|
475
|
+
/**
|
|
476
|
+
* Control-frame `type` values in codex's `--json` event stream. None of them
|
|
477
|
+
* carry the model's answer.
|
|
478
|
+
*/
|
|
479
|
+
const STREAM_CONTROL_TYPES = new Set([
|
|
480
|
+
'thread.started', 'turn.started', 'turn.completed', 'turn.failed',
|
|
481
|
+
'item.started', 'item.updated', 'item.completed', 'error',
|
|
482
|
+
]);
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Detects a failure the exit code did not report.
|
|
486
|
+
*
|
|
487
|
+
* CLI agents exit 0 on failed runs and describe the failure only in the
|
|
488
|
+
* payload (verified 2026-08-01):
|
|
489
|
+
* claude → {"subtype":"success","is_error":true,"result":"Not logged in …"}
|
|
490
|
+
* codex → {"type":"turn.failed","error":{"message":"…"}}
|
|
491
|
+
* agy → {"status":"ERROR"|"CANCELED","error":"…"}
|
|
492
|
+
*
|
|
493
|
+
* Without this, `status === 0` accepts the error text as the model's answer —
|
|
494
|
+
* e.g. "Not logged in · Please run /login" gets stored as a memory node.
|
|
495
|
+
*
|
|
496
|
+
* @param {string} output raw stdout
|
|
497
|
+
* @returns {string|null} failure reason, or null if the run looks clean
|
|
498
|
+
*/
|
|
499
|
+
function detectAgentFailure(output) {
|
|
500
|
+
if (!output) return null;
|
|
501
|
+
for (const line of output.split('\n')) {
|
|
502
|
+
const t = line.trim();
|
|
503
|
+
if (!t.startsWith('{')) continue;
|
|
504
|
+
let parsed;
|
|
505
|
+
try {
|
|
506
|
+
parsed = JSON.parse(t);
|
|
507
|
+
} catch {
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
if (parsed.is_error === true) {
|
|
511
|
+
return typeof parsed.result === 'string' ? parsed.result : 'is_error: true';
|
|
512
|
+
}
|
|
513
|
+
if (parsed.type === 'turn.failed' || parsed.type === 'error') {
|
|
514
|
+
const msg = parsed.error?.message ?? parsed.message;
|
|
515
|
+
return typeof msg === 'string' ? msg : String(parsed.type);
|
|
516
|
+
}
|
|
517
|
+
if (typeof parsed.status === 'string' && parsed.status.toUpperCase() !== 'SUCCESS') {
|
|
518
|
+
return typeof parsed.error === 'string'
|
|
519
|
+
? `${parsed.status}: ${parsed.error}`
|
|
520
|
+
: `status=${parsed.status}`;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
return null;
|
|
524
|
+
}
|
|
525
|
+
|
|
409
526
|
function parseAgentOutput(output) {
|
|
410
527
|
try {
|
|
411
528
|
const parsed = JSON.parse(output);
|
|
@@ -418,7 +535,34 @@ function parseAgentOutput(output) {
|
|
|
418
535
|
return last.content || last.text || last.response || JSON.stringify(last);
|
|
419
536
|
}
|
|
420
537
|
} catch {
|
|
421
|
-
// Not JSON
|
|
538
|
+
// Not a single JSON document. It may still be a JSONL event stream —
|
|
539
|
+
// codex `--json` emits one frame per line, so a whole-output JSON.parse
|
|
540
|
+
// always throws and the raw stream used to be returned as the "answer".
|
|
541
|
+
const answers = [];
|
|
542
|
+
for (const line of output.split('\n')) {
|
|
543
|
+
const t = line.trim();
|
|
544
|
+
if (!t.startsWith('{')) continue;
|
|
545
|
+
let frame;
|
|
546
|
+
try {
|
|
547
|
+
frame = JSON.parse(t);
|
|
548
|
+
} catch {
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
// codex: the reply is item.text of the `agent_message` item.
|
|
552
|
+
if (frame.item?.type === 'agent_message' && typeof frame.item.text === 'string') {
|
|
553
|
+
answers.push(frame.item.text);
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
if (STREAM_CONTROL_TYPES.has(frame.type)) continue;
|
|
557
|
+
for (const key of ['response', 'result', 'content', 'text']) {
|
|
558
|
+
if (typeof frame[key] === 'string') {
|
|
559
|
+
answers.push(frame[key]);
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
// Last message wins: earlier frames are partial or superseded.
|
|
565
|
+
if (answers.length > 0) return answers[answers.length - 1];
|
|
422
566
|
}
|
|
423
567
|
return output;
|
|
424
568
|
}
|
package/src/core/search.mjs
CHANGED
|
@@ -132,12 +132,22 @@ export async function semanticSearch(query, {
|
|
|
132
132
|
const filteredSlugs = new Set(filteredNodes.map(n => n.slug));
|
|
133
133
|
const results = [];
|
|
134
134
|
|
|
135
|
-
// Get query embedding
|
|
135
|
+
// Get query embedding.
|
|
136
|
+
//
|
|
137
|
+
// Falling back to lexical-only search is fine. Losing the REASON is not:
|
|
138
|
+
// "the embedding API failed" and "the index for this brain was never built"
|
|
139
|
+
// are completely different problems with completely different fixes, and
|
|
140
|
+
// this used to report both as the former. That mislabelling cost real time —
|
|
141
|
+
// festech.live's project brain had 2602 vault nodes and 0 embeddings, and
|
|
142
|
+
// every recall against it printed "Embedding API unreachable or
|
|
143
|
+
// unconfigured. Configure GOOGLE_API_KEY or OPENAI_API_KEY", which sent the
|
|
144
|
+
// reader off to check credentials that were working perfectly.
|
|
136
145
|
let queryEmbedding = null;
|
|
146
|
+
let embeddingError = null;
|
|
137
147
|
try {
|
|
138
148
|
queryEmbedding = await getEmbedding(String(query));
|
|
139
149
|
} catch (err) {
|
|
140
|
-
|
|
150
|
+
embeddingError = String(err?.message || err);
|
|
141
151
|
}
|
|
142
152
|
|
|
143
153
|
const vaultIndex = loadEmbeddingsIndex(derivedDir);
|
|
@@ -279,6 +289,13 @@ export async function semanticSearch(query, {
|
|
|
279
289
|
const finalResults = results.slice(0, k);
|
|
280
290
|
if (!queryEmbedding || vaultEntries.length === 0) {
|
|
281
291
|
finalResults.degradedTextSearch = true;
|
|
292
|
+
// Say WHICH failure this is. The caller renders remediation from it, and
|
|
293
|
+
// the two causes need opposite fixes: one is credentials, the other is an
|
|
294
|
+
// unbuilt index for this specific brain.
|
|
295
|
+
finalResults.degradedReason = !queryEmbedding ? 'embedding_failed' : 'empty_index';
|
|
296
|
+
finalResults.degradedDetail = !queryEmbedding
|
|
297
|
+
? (embeddingError || 'embedding provider returned nothing')
|
|
298
|
+
: `no embeddings indexed for this brain (${derivedDir})`;
|
|
282
299
|
}
|
|
283
300
|
|
|
284
301
|
return finalResults;
|
|
@@ -552,6 +552,23 @@ export function resetGateStats() {
|
|
|
552
552
|
* Reset internal rate-limit/concurrency state (for testing only).
|
|
553
553
|
*/
|
|
554
554
|
export function resetGateStateForTests() {
|
|
555
|
+
// Tear down policy watchers FIRST. They are module-level singletons that
|
|
556
|
+
// outlive an individual test, and their callbacks call applyPolicy() +
|
|
557
|
+
// drainQueue() — so a watcher registered by an earlier test can fire in the
|
|
558
|
+
// middle of a later one, silently rewriting blockedDomains/domainMinInterval
|
|
559
|
+
// and draining the queue. That made the firewall-reject and minInterval
|
|
560
|
+
// tests fail intermittently under full-suite load while passing in
|
|
561
|
+
// isolation. Closing them here makes each test start from a quiet state.
|
|
562
|
+
for (const w of [policyWatcher, parentWatcher]) {
|
|
563
|
+
try {
|
|
564
|
+
w?.close();
|
|
565
|
+
} catch {
|
|
566
|
+
// already closed / never opened
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
policyWatcher = null;
|
|
570
|
+
parentWatcher = null;
|
|
571
|
+
|
|
555
572
|
gateGeneration += 1;
|
|
556
573
|
globalInFlight = 0;
|
|
557
574
|
domainInFlight.clear();
|
package/src/server/index.mjs
CHANGED
|
@@ -187,8 +187,55 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
|
|
|
187
187
|
// Check daemon status
|
|
188
188
|
const daemonStatus = getDaemonStatus();
|
|
189
189
|
|
|
190
|
+
// Embedding coverage per brain.
|
|
191
|
+
//
|
|
192
|
+
// This existed nowhere before, and its absence is exactly why a brain with
|
|
193
|
+
// 2602 vault nodes and 0 embeddings looked healthy for weeks: compile
|
|
194
|
+
// reported `compiled: true`, recall returned keyword hits that read like
|
|
195
|
+
// real results, and nothing anywhere printed "N of M nodes embedded".
|
|
196
|
+
// Vector search silently off is a CRITICAL failure, not a cosmetic one —
|
|
197
|
+
// recall answers from the wrong brain and the caller cannot tell.
|
|
198
|
+
const embeddingCoverage = [];
|
|
199
|
+
try {
|
|
200
|
+
const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
|
|
201
|
+
const { getActiveBrains } = await import('../core/config.mjs');
|
|
202
|
+
const brains = getActiveBrains();
|
|
203
|
+
for (const b of Object.values(brains)) {
|
|
204
|
+
if (!b?.brainDir) continue;
|
|
205
|
+
const vaultPath = path.join(b.brainDir, 'memory-vault');
|
|
206
|
+
if (!fs.existsSync(vaultPath)) continue;
|
|
207
|
+
let nodeCount = 0;
|
|
208
|
+
const walk = (d) => {
|
|
209
|
+
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
|
210
|
+
if (e.name.startsWith('.')) continue;
|
|
211
|
+
const p = path.join(d, e.name);
|
|
212
|
+
if (e.isDirectory()) walk(p);
|
|
213
|
+
else if (e.name.endsWith('.md')) nodeCount++;
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
try { walk(vaultPath); } catch { /* unreadable vault */ }
|
|
217
|
+
let embedded = 0;
|
|
218
|
+
try {
|
|
219
|
+
embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
|
|
220
|
+
} catch { /* index unreadable — reported as 0, which is the honest value */ }
|
|
221
|
+
embeddingCoverage.push({
|
|
222
|
+
layer: b.layer || 'unknown',
|
|
223
|
+
nodes: nodeCount,
|
|
224
|
+
embedded,
|
|
225
|
+
vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
} catch (err) {
|
|
229
|
+
embeddingCoverage.push({ error: String(err?.message || err) });
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// A brain with nodes but no vectors answers every query from keyword
|
|
233
|
+
// matching. Surface it at the same severity as a dead daemon.
|
|
234
|
+
const unembeddedBrains = embeddingCoverage.filter(c => c.nodes > 0 && c.embedded === 0);
|
|
235
|
+
|
|
190
236
|
// Determine overall status
|
|
191
|
-
const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
|
|
237
|
+
const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
|
|
238
|
+
|| unembeddedBrains.length > 0;
|
|
192
239
|
|
|
193
240
|
// Check Caddy and Cloudflare status
|
|
194
241
|
let caddyStatus = 'inactive';
|
|
@@ -223,6 +270,12 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
|
|
|
223
270
|
disk,
|
|
224
271
|
cli_agents: cliAgents,
|
|
225
272
|
daemon: daemonStatus,
|
|
273
|
+
embedding_coverage: embeddingCoverage,
|
|
274
|
+
// Explicit, greppable signal. A brain with nodes and no vectors serves
|
|
275
|
+
// keyword-only results that look exactly like real ones.
|
|
276
|
+
vector_search_degraded: unembeddedBrains.length > 0
|
|
277
|
+
? unembeddedBrains.map(b => `${b.layer}: ${b.embedded}/${b.nodes} embedded`)
|
|
278
|
+
: null,
|
|
226
279
|
caddy: caddyStatus,
|
|
227
280
|
cloudflare: cloudflareStatus,
|
|
228
281
|
emergency_alerts: emergencyAlerts || null,
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { compileSurface } from '../../core/surface.mjs';
|
|
16
16
|
import { getNodes, invalidate } from '../../core/vault-cache.mjs';
|
|
17
17
|
import { loadEmbeddingsIndex, loadSessionEmbeddingsIndex } from '../../core/embeddings.mjs';
|
|
18
|
+
import { logger } from '../../core/logger.mjs';
|
|
18
19
|
|
|
19
20
|
const router = Router();
|
|
20
21
|
|
|
@@ -32,20 +33,30 @@ router.post('/api/vault/compile', requireAuth, requireScope('memory:recompile'),
|
|
|
32
33
|
force: true,
|
|
33
34
|
});
|
|
34
35
|
// Incrementally embed any new vault nodes and sessions for THIS brain
|
|
35
|
-
let vaultEmbed = null, sessionEmbed = null;
|
|
36
|
+
let vaultEmbed = null, sessionEmbed = null, embedError = null;
|
|
36
37
|
try {
|
|
37
38
|
const { buildEmbeddingsIndex, buildSessionEmbeddingsIndex } = await import('../../core/embeddings.mjs');
|
|
38
39
|
invalidate(vaultDir);
|
|
39
40
|
const vaultNodes = getNodes(vaultDir);
|
|
40
41
|
vaultEmbed = await buildEmbeddingsIndex(vaultNodes, paths.derivedDir);
|
|
41
42
|
sessionEmbed = await buildSessionEmbeddingsIndex(paths.sessionsDir, paths.derivedDir);
|
|
42
|
-
} catch {
|
|
43
|
+
} catch (err) {
|
|
44
|
+
// Still non-fatal — a compile that produced surfaces is worth keeping —
|
|
45
|
+
// but NEVER silent. This was a bare `catch {}`, and it meant every
|
|
46
|
+
// compile could fail to embed a single node while reporting
|
|
47
|
+
// `compiled: true`. A brain with thousands of nodes and zero vectors
|
|
48
|
+
// looked perfectly healthy, and recall silently answered from keyword
|
|
49
|
+
// matching for weeks.
|
|
50
|
+
embedError = String(err?.message || err);
|
|
51
|
+
logger.error('vault', `Embedding build FAILED during compile (surfaces still compiled): ${embedError}`, { vaultDir });
|
|
52
|
+
}
|
|
43
53
|
res.json({
|
|
44
54
|
compiled: true,
|
|
45
55
|
elapsed_ms: Date.now() - start,
|
|
46
56
|
vault_dir: vaultDir,
|
|
47
57
|
vault_embeddings: vaultEmbed,
|
|
48
58
|
session_embeddings: sessionEmbed,
|
|
59
|
+
embedding_error: embedError,
|
|
49
60
|
});
|
|
50
61
|
} catch (err) {
|
|
51
62
|
serverError(res, err);
|