peon-mem 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +301 -0
- package/bin/peon-mem.mjs +273 -0
- package/dist/brain.d.ts +72 -0
- package/dist/brain.js +224 -0
- package/dist/compression.d.ts +9 -0
- package/dist/compression.js +37 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.js +99 -0
- package/dist/daemon-cli.d.ts +2 -0
- package/dist/daemon-cli.js +54 -0
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +1078 -0
- package/dist/embedding-store.d.ts +43 -0
- package/dist/embedding-store.js +169 -0
- package/dist/embeddings.d.ts +93 -0
- package/dist/embeddings.js +345 -0
- package/dist/entities.d.ts +61 -0
- package/dist/entities.js +191 -0
- package/dist/entity-extraction.d.ts +33 -0
- package/dist/entity-extraction.js +75 -0
- package/dist/eval-metrics.d.ts +27 -0
- package/dist/eval-metrics.js +50 -0
- package/dist/evaluation.d.ts +58 -0
- package/dist/evaluation.js +244 -0
- package/dist/global-extraction.d.ts +15 -0
- package/dist/global-extraction.js +61 -0
- package/dist/global-memory.d.ts +43 -0
- package/dist/global-memory.js +306 -0
- package/dist/global-promotion.d.ts +25 -0
- package/dist/global-promotion.js +29 -0
- package/dist/hyde.d.ts +31 -0
- package/dist/hyde.js +46 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +246 -0
- package/dist/injection.d.ts +38 -0
- package/dist/injection.js +133 -0
- package/dist/logger.d.ts +17 -0
- package/dist/logger.js +63 -0
- package/dist/memory-mutations.d.ts +24 -0
- package/dist/memory-mutations.js +57 -0
- package/dist/memory-store.d.ts +194 -0
- package/dist/memory-store.js +1205 -0
- package/dist/monitor.d.ts +13 -0
- package/dist/monitor.js +977 -0
- package/dist/overview.d.ts +73 -0
- package/dist/overview.js +104 -0
- package/dist/processor.d.ts +90 -0
- package/dist/processor.js +450 -0
- package/dist/quality.d.ts +86 -0
- package/dist/quality.js +338 -0
- package/dist/recuration.d.ts +13 -0
- package/dist/recuration.js +65 -0
- package/dist/reranker.d.ts +34 -0
- package/dist/reranker.js +89 -0
- package/dist/retrieval.d.ts +106 -0
- package/dist/retrieval.js +392 -0
- package/dist/session-index.d.ts +34 -0
- package/dist/session-index.js +87 -0
- package/dist/temporal.d.ts +20 -0
- package/dist/temporal.js +62 -0
- package/dist/token-ab-monitor.d.ts +1 -0
- package/dist/token-ab-monitor.js +7 -0
- package/dist/tools.d.ts +232 -0
- package/dist/tools.js +546 -0
- package/dist/types.d.ts +169 -0
- package/dist/types.js +1 -0
- package/docs/assets/neural-universe.png +0 -0
- package/package.json +57 -0
- package/scripts/claude-peon-hook.mjs +522 -0
- package/scripts/codex-peon-hook.mjs +4 -0
- package/scripts/eval-retrieval-labeled.mjs +135 -0
- package/scripts/eval-retrieval.mjs +96 -0
- package/scripts/evaluate-peon.mjs +47 -0
- package/scripts/install-peon-stl.mjs +82 -0
- package/scripts/install-peon.mjs +318 -0
- package/scripts/lib/eval-ledger.mjs +104 -0
- package/scripts/lib/stl-classify.mjs +44 -0
- package/scripts/longmemeval-eval.mjs +144 -0
- package/scripts/peon-report.mjs +155 -0
- package/scripts/peon-stl.mjs +506 -0
- package/scripts/token-ab-monitor.html +235 -0
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { createReadStream as fsCreateReadStream, existsSync } from "node:fs";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { dirname, join } from "node:path";
|
|
6
|
+
import { createInterface } from "node:readline";
|
|
7
|
+
|
|
8
|
+
// Hard off-switch (for A/B testing). When PEON_DISABLED is set, the hook does
|
|
9
|
+
// nothing at all — no recording, no context injection, no daemon calls — so the
|
|
10
|
+
// session runs purely on the cloud model with zero Peon involvement.
|
|
11
|
+
if (/^(1|true|yes|on)$/i.test(process.env.PEON_DISABLED || "")) process.exit(0);
|
|
12
|
+
|
|
13
|
+
const daemonUrl = (process.env.PEON_DAEMON_URL || "http://127.0.0.1:3737").replace(/\/$/, "");
|
|
14
|
+
const stateDir =
|
|
15
|
+
process.env.PEON_HOOK_STATE_DIR || join(homedir(), "Library", "Application Support", "Peon", "claude-hooks");
|
|
16
|
+
const hookClient = process.env.PEON_HOOK_CLIENT || "claude-code";
|
|
17
|
+
|
|
18
|
+
// Framing that makes injected memory authoritative — so the model consults it
|
|
19
|
+
// FIRST instead of defaulting to a web search or re-reading files. Without this,
|
|
20
|
+
// the memory is just passive context and gets ignored. Declared up here (not by
|
|
21
|
+
// the formatters) so it's initialized before the top-level await block runs.
|
|
22
|
+
const PEON_FIRST_DIRECTIVE =
|
|
23
|
+
"<peon-memory>\nThis is your saved memory for THIS project and the user's environment, recalled automatically by Peon. " +
|
|
24
|
+
"Treat it as AUTHORITATIVE and consult it FIRST. If it already answers what the user is asking — a PROCEDURE (how to " +
|
|
25
|
+
"run/build/submit/deploy something), a past RESULT or number, a FILE PATH, or a DECISION/config — use that answer " +
|
|
26
|
+
"directly and cite it. Do NOT re-run a command, re-submit or re-explore a cluster/job, re-read source, or re-derive " +
|
|
27
|
+
"something JUST to rediscover what is already stated here (e.g. don't go re-inspect how to run an experiment the memory " +
|
|
28
|
+
"already documents). Go to the source ONLY to genuinely verify when the memory is absent, stale, or the user explicitly " +
|
|
29
|
+
"asks for fresh verification. For anything relevant not shown below, call the Peon `search_memory` tool before falling " +
|
|
30
|
+
"back to reading code or re-running work. You do not need to be asked to use this memory.\n</peon-memory>";
|
|
31
|
+
|
|
32
|
+
// The retrieval query rides on the GET /context request line as a URL search
|
|
33
|
+
// param. A large user prompt (a pasted JD, a big code block) URL-encodes into a
|
|
34
|
+
// request line that overflows the daemon's Node http server default max header
|
|
35
|
+
// size (~16KB) → the server answers 431 before the handler runs, and injection
|
|
36
|
+
// silently fails. The first N chars are more than enough to rank retrieval, so
|
|
37
|
+
// cap at the single choke point getProjectContext (sibling callers already slice
|
|
38
|
+
// to 400/600, well under this). Declared up here — like PEON_FIRST_DIRECTIVE —
|
|
39
|
+
// so it is initialized before the top-level await block calls getProjectContext.
|
|
40
|
+
const MAX_CONTEXT_QUERY_CHARS = 2000;
|
|
41
|
+
|
|
42
|
+
// Resolve any working directory to its ONE project brain, so memory never fragments:
|
|
43
|
+
// 1. collapse git-worktree paths to the repo root (…/.claude/worktrees/x → repo)
|
|
44
|
+
// 2. walk UP to the TOPMOST ancestor that already holds a Peon brain (.peon), bounded by
|
|
45
|
+
// home. A project with no .git (e.g. "Master Project 700B") would otherwise spawn a
|
|
46
|
+
// separate empty brain in every subfolder a session starts in — so working inside
|
|
47
|
+
// Privacy_NL2SQL/ injected nothing while the real 1400+ belief brain sat at the root.
|
|
48
|
+
// Topmost-.peon unifies all subfolders onto the root brain.
|
|
49
|
+
// 3. EXCEPT a `.peon/root` marker declares a brain BOUNDARY: the nearest one wins and the climb
|
|
50
|
+
// stops there, so a big sub-project (e.g. a thesis folder) keeps its OWN brain. Must match
|
|
51
|
+
// canonicalProjectPath() in src/daemon.ts so hook + direct-MCP + Codex all resolve identically.
|
|
52
|
+
function resolveProjectPath(p) {
|
|
53
|
+
const marker = "/.claude/worktrees/";
|
|
54
|
+
const idx = p.indexOf(marker);
|
|
55
|
+
const base = idx !== -1 ? p.slice(0, idx) : p;
|
|
56
|
+
const home = homedir();
|
|
57
|
+
let dir = base;
|
|
58
|
+
let rootBrain = null;
|
|
59
|
+
while (dir && dir.startsWith(home) && dir !== home) {
|
|
60
|
+
if (existsSync(join(dir, ".peon"))) {
|
|
61
|
+
if (existsSync(join(dir, ".peon", "root"))) return dir; // boundary marker — its own brain
|
|
62
|
+
rootBrain = dir; // otherwise topmost-wins
|
|
63
|
+
}
|
|
64
|
+
const parent = dirname(dir);
|
|
65
|
+
if (!parent || parent === dir) break;
|
|
66
|
+
dir = parent;
|
|
67
|
+
}
|
|
68
|
+
return rootBrain || base;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
let eventName = "unknown";
|
|
72
|
+
let projectPath = resolveProjectPath(process.env.CLAUDE_PROJECT_DIR || process.env.CODEX_PROJECT_DIR || process.cwd());
|
|
73
|
+
let externalSessionId = process.env.CLAUDE_CODE_SESSION_ID || process.env.CODEX_SESSION_ID || "unknown-session";
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
const input = await readStdinJson();
|
|
77
|
+
eventName = normalizeEventName(readFirst(input, ["hook_event_name", "event_name", "eventName", "event", "name", "type"]));
|
|
78
|
+
projectPath = resolveProjectPath(readText(input, [
|
|
79
|
+
"cwd",
|
|
80
|
+
"project.cwd",
|
|
81
|
+
"workspace.cwd",
|
|
82
|
+
"workspace_path",
|
|
83
|
+
"workspacePath",
|
|
84
|
+
"project_path",
|
|
85
|
+
"projectPath"
|
|
86
|
+
]) || projectPath);
|
|
87
|
+
externalSessionId =
|
|
88
|
+
readText(input, ["session_id", "sessionId", "conversation_id", "conversationId", "thread_id", "threadId"]) ||
|
|
89
|
+
externalSessionId;
|
|
90
|
+
|
|
91
|
+
if (eventName === "SessionStart") {
|
|
92
|
+
await ensurePeonSession({ projectPath, externalSessionId, client: hookClient });
|
|
93
|
+
const context = await getProjectContext(projectPath, "recent project context decisions artifacts current work");
|
|
94
|
+
const startupContext = formatStartupContext(context);
|
|
95
|
+
if (startupContext) process.stdout.write(startupContext);
|
|
96
|
+
} else if (eventName === "SubagentStart") {
|
|
97
|
+
// Subagents DON'T fire SessionStart/UserPromptSubmit, so without this they start
|
|
98
|
+
// blind and redo work already in memory (the multi-agent token-waste problem).
|
|
99
|
+
// Give each worker the same Peon-first memory the main session gets — and pull
|
|
100
|
+
// CROSS-project, since a worker may touch a paper that lives in another brain.
|
|
101
|
+
const task = extractText(
|
|
102
|
+
readFirst(input, ["prompt", "description", "task", "agent_prompt", "input", "message"])
|
|
103
|
+
);
|
|
104
|
+
const memory = await buildSubagentContext(task, projectPath);
|
|
105
|
+
if (memory) {
|
|
106
|
+
process.stdout.write(
|
|
107
|
+
JSON.stringify({ hookSpecificOutput: { hookEventName: "SubagentStart", additionalContext: memory } })
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
} else if (eventName === "SubagentStop") {
|
|
111
|
+
// Mirror of SubagentStart: capture what the worker PRODUCED so it's never redone.
|
|
112
|
+
// Subagent tool-uses don't route to the main session, so this is the only place
|
|
113
|
+
// their findings enter Peon.
|
|
114
|
+
const result = extractText(
|
|
115
|
+
readFirst(input, ["result", "output", "final_output", "response", "summary", "last_assistant_message", "message"])
|
|
116
|
+
);
|
|
117
|
+
const agentType = extractText(readFirst(input, ["agent_type", "subagent_type", "matcher", "name"]));
|
|
118
|
+
if (result.trim()) {
|
|
119
|
+
await recordWithSession({ projectPath, externalSessionId, client: hookClient }, (sessionId) =>
|
|
120
|
+
postJson("/events", { sessionId, type: "assistant_summary", content: `[subagent${agentType ? ":" + agentType : ""}] ${result.slice(0, 2000)}` }));
|
|
121
|
+
await postJson("/process/auto", { projectPath, trigger: "subagent_end" }).catch(() => undefined);
|
|
122
|
+
}
|
|
123
|
+
} else if (eventName === "PreToolUse") {
|
|
124
|
+
// Only wired for WebSearch/WebFetch (matcher-scoped in settings). Before an
|
|
125
|
+
// expensive web call, surface what Peon already knows so it isn't re-fetched.
|
|
126
|
+
const toolName = extractText(readFirst(input, ["tool_name", "toolName", "tool.name", "tool", "name"]));
|
|
127
|
+
const q = extractText(readFirst(input, ["tool_input.query", "tool_input.prompt", "tool_input.url", "input.query", "input.url", "query", "url"]))
|
|
128
|
+
|| summarize(readFirst(input, ["tool_input", "input", "arguments"]) || {});
|
|
129
|
+
if (/^(WebSearch|WebFetch)$/i.test(toolName) && q.trim()) {
|
|
130
|
+
const hits = await getCrossProjectRecall(q.slice(0, 400), null).catch(() => "");
|
|
131
|
+
const local = await getProjectContext(projectPath, q.slice(0, 400)).then((c) => formatRelevantMemory(c)).catch(() => "");
|
|
132
|
+
const note = [local, hits].filter(Boolean).join("\n\n");
|
|
133
|
+
if (note) {
|
|
134
|
+
process.stdout.write(JSON.stringify({
|
|
135
|
+
hookSpecificOutput: { hookEventName: "PreToolUse", additionalContext: `Before searching the web, note Peon already has related memory — reuse it if it answers the need:\n${note}` }
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
} else if (eventName === "UserPromptSubmit" || eventName === "UserPromptExpansion") {
|
|
140
|
+
const prompt = extractText(readFirst(input, ["prompt", "message", "user_prompt", "userPrompt", "input", "expanded_prompt", "command"]));
|
|
141
|
+
if (prompt.trim()) {
|
|
142
|
+
const context = await getProjectContext(projectPath, prompt);
|
|
143
|
+
const relevantMemory = formatRelevantMemory(context);
|
|
144
|
+
if (relevantMemory) {
|
|
145
|
+
// UserPromptExpansion expects structured additionalContext; UserPromptSubmit takes raw stdout.
|
|
146
|
+
if (eventName === "UserPromptExpansion") {
|
|
147
|
+
process.stdout.write(JSON.stringify({ hookSpecificOutput: { hookEventName: "UserPromptExpansion", additionalContext: relevantMemory } }));
|
|
148
|
+
} else {
|
|
149
|
+
process.stdout.write(relevantMemory);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
await recordWithSession({ projectPath, externalSessionId, client: hookClient }, (sessionId) =>
|
|
153
|
+
postJson("/messages", { sessionId, role: "user", content: prompt }));
|
|
154
|
+
}
|
|
155
|
+
} else if (eventName === "PostToolUse" || eventName === "PostToolUseFailure") {
|
|
156
|
+
await recordWithSession({ projectPath, externalSessionId, client: hookClient }, (sessionId) =>
|
|
157
|
+
postJson("/events", { sessionId, type: "tool_use", content: formatToolUse(input) }));
|
|
158
|
+
} else if (eventName === "Stop") {
|
|
159
|
+
// A response TURN finished — capture the summary and let Peon consolidate,
|
|
160
|
+
// but keep the session ALIVE. Stop fires after every turn; the Claude session
|
|
161
|
+
// (and the Peon session) continues across turns and only ends on SessionEnd.
|
|
162
|
+
const finalMessage = extractAssistantSummary(input);
|
|
163
|
+
if (finalMessage.trim()) {
|
|
164
|
+
await recordWithSession({ projectPath, externalSessionId, client: hookClient }, (sessionId) =>
|
|
165
|
+
postJson("/events", { sessionId, type: "assistant_summary", content: finalMessage.slice(0, 2000) }));
|
|
166
|
+
}
|
|
167
|
+
await postJson("/process/auto", { projectPath, trigger: "turn_end" }).catch(() => undefined);
|
|
168
|
+
await trackTokenUsage(input, projectPath, externalSessionId);
|
|
169
|
+
} else if (eventName === "SessionEnd") {
|
|
170
|
+
const finalMessage = extractAssistantSummary(input);
|
|
171
|
+
if (finalMessage.trim()) {
|
|
172
|
+
await recordWithSession({ projectPath, externalSessionId, client: hookClient }, (sessionId) =>
|
|
173
|
+
postJson("/events", { sessionId, type: "assistant_summary", content: finalMessage.slice(0, 2000) })).catch(() => undefined);
|
|
174
|
+
}
|
|
175
|
+
const session = await readSession(externalSessionId);
|
|
176
|
+
try {
|
|
177
|
+
if (session?.sessionId) await postJson(`/sessions/${encodeURIComponent(session.sessionId)}/end`, {});
|
|
178
|
+
} finally {
|
|
179
|
+
// Always clear the local cache, even if the end-call failed — otherwise a
|
|
180
|
+
// stale session id gets replayed forever and recording silently dies.
|
|
181
|
+
await removeSession(externalSessionId);
|
|
182
|
+
}
|
|
183
|
+
await trackTokenUsage(input, projectPath, externalSessionId);
|
|
184
|
+
}
|
|
185
|
+
} catch (error) {
|
|
186
|
+
await appendLocalError({
|
|
187
|
+
createdAt: new Date().toISOString(),
|
|
188
|
+
eventName,
|
|
189
|
+
projectPath,
|
|
190
|
+
externalSessionId,
|
|
191
|
+
error: error instanceof Error ? error.message : String(error)
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async function ensurePeonSession({ projectPath, externalSessionId, client }) {
|
|
196
|
+
const existing = await readSession(externalSessionId);
|
|
197
|
+
if (existing?.projectPath === projectPath && existing.sessionId) return existing.sessionId;
|
|
198
|
+
const started = await postJson("/sessions", { projectPath, client, cwd: projectPath });
|
|
199
|
+
await writeSession(externalSessionId, {
|
|
200
|
+
sessionId: started.sessionId,
|
|
201
|
+
projectPath,
|
|
202
|
+
client,
|
|
203
|
+
startedAt: new Date().toISOString()
|
|
204
|
+
});
|
|
205
|
+
return started.sessionId;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Record through the project's Peon session, self-healing if the daemon has
|
|
210
|
+
* forgotten it (restart, or the session was ended). On "Unknown Peon session"
|
|
211
|
+
* we drop the stale cache, recreate the session, and retry once — so a daemon
|
|
212
|
+
* bounce can never silently break recording for the rest of the Claude session.
|
|
213
|
+
*/
|
|
214
|
+
async function recordWithSession(ctx, run) {
|
|
215
|
+
const sessionId = await ensurePeonSession(ctx);
|
|
216
|
+
try {
|
|
217
|
+
return await run(sessionId);
|
|
218
|
+
} catch (error) {
|
|
219
|
+
if (!/Unknown Peon session/i.test(String(error && error.message))) throw error;
|
|
220
|
+
await removeSession(ctx.externalSessionId);
|
|
221
|
+
const fresh = await ensurePeonSession(ctx);
|
|
222
|
+
return await run(fresh);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async function trackTokenUsage(input, projectPath, externalSessionId) {
|
|
227
|
+
const transcriptPath = readText(input, ["transcript_path", "transcriptPath"]);
|
|
228
|
+
if (transcriptPath) await recordTokenUsage({ projectPath, externalSessionId, transcriptPath });
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function formatToolUse(input) {
|
|
232
|
+
const toolName = extractText(readFirst(input, ["tool_name", "toolName", "tool.name", "tool", "name"])) || "unknown";
|
|
233
|
+
const toolInput = summarize(readFirst(input, ["tool_input", "toolInput", "input", "arguments", "args"]) || {});
|
|
234
|
+
const toolResponse = summarize(readFirst(input, ["tool_response", "toolResponse", "response", "result", "output"]) || {});
|
|
235
|
+
return [`Tool used: ${toolName}`, `Input: ${toolInput}`, `Output: ${toolResponse}`].join("\n");
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function summarize(value) {
|
|
239
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
240
|
+
return text.length > 1200 ? `${text.slice(0, 1200)}...` : text;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function postJson(path, body) {
|
|
244
|
+
const response = await fetch(`${daemonUrl}${path}`, {
|
|
245
|
+
method: "POST",
|
|
246
|
+
headers: { "content-type": "application/json" },
|
|
247
|
+
body: JSON.stringify(body)
|
|
248
|
+
});
|
|
249
|
+
const text = await response.text();
|
|
250
|
+
if (!response.ok) throw new Error(`${path} failed with ${response.status}: ${text}`);
|
|
251
|
+
return text ? JSON.parse(text) : {};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async function getProjectContext(projectPath, query) {
|
|
255
|
+
const url = new URL(`${daemonUrl}/context`);
|
|
256
|
+
url.searchParams.set("projectPath", projectPath);
|
|
257
|
+
url.searchParams.set("query", String(query ?? "").slice(0, MAX_CONTEXT_QUERY_CHARS));
|
|
258
|
+
url.searchParams.set("maxChars", "6000");
|
|
259
|
+
const response = await fetch(url);
|
|
260
|
+
const text = await response.text();
|
|
261
|
+
if (!response.ok) throw new Error(`/context failed with ${response.status}: ${text}`);
|
|
262
|
+
return text ? JSON.parse(text) : {};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Build the memory packet handed to a subagent at spawn. Subagents work on tasks
|
|
267
|
+
* that may live in OTHER project brains (e.g. a worker extracting from the DTS-SQL
|
|
268
|
+
* paper while the main session is in Master Project 700B), so this pulls the
|
|
269
|
+
* current project's context AND a cross-project recall for the worker's task —
|
|
270
|
+
* so workers don't redo work already captured anywhere in Peon.
|
|
271
|
+
*/
|
|
272
|
+
async function buildSubagentContext(task, projectPath) {
|
|
273
|
+
const query = (task || "recent project context decisions artifacts current work").slice(0, 600);
|
|
274
|
+
let local = "";
|
|
275
|
+
try {
|
|
276
|
+
local = formatRelevantMemory(await getProjectContext(projectPath, query)); // already carries the directive
|
|
277
|
+
} catch { /* best effort */ }
|
|
278
|
+
let cross = "";
|
|
279
|
+
try {
|
|
280
|
+
cross = await getCrossProjectRecall(query, projectPath);
|
|
281
|
+
} catch { /* best effort */ }
|
|
282
|
+
if (!local && !cross) return "";
|
|
283
|
+
if (local) return cross ? `${local}\n\n${cross}` : local;
|
|
284
|
+
return `${PEON_FIRST_DIRECTIVE}\n${cross}`; // cross-only path needs the directive
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function getCrossProjectRecall(query, excludeProjectPath) {
|
|
288
|
+
const url = new URL(`${daemonUrl}/cross-context`);
|
|
289
|
+
url.searchParams.set("query", query);
|
|
290
|
+
url.searchParams.set("limit", "10");
|
|
291
|
+
if (excludeProjectPath) url.searchParams.set("exclude", excludeProjectPath);
|
|
292
|
+
const response = await fetch(url);
|
|
293
|
+
if (!response.ok) return "";
|
|
294
|
+
const data = JSON.parse((await response.text()) || "{}");
|
|
295
|
+
const hits = (data.results || []).filter((h) => h && h.record && h.record.content);
|
|
296
|
+
if (hits.length === 0) return "";
|
|
297
|
+
const lines = hits
|
|
298
|
+
.slice(0, 10)
|
|
299
|
+
.map((h) => `- [${h.projectName}] ${String(h.record.content).slice(0, 200)}`)
|
|
300
|
+
.join("\n");
|
|
301
|
+
return `From your OTHER projects (work already done elsewhere — reuse it, don't redo it):\n${lines}`;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function formatRelevantMemory(context) {
|
|
305
|
+
const sections = [
|
|
306
|
+
["Peon Global Brain (applies to every project)", context.global],
|
|
307
|
+
["Summary", context.summary],
|
|
308
|
+
["Memory Records", context.memories],
|
|
309
|
+
["Decisions", context.decisions],
|
|
310
|
+
["Preferences", context.preferences],
|
|
311
|
+
["Open Questions", context.openQuestions],
|
|
312
|
+
["Artifacts", context.artifacts],
|
|
313
|
+
["Recent Timeline", context.timeline]
|
|
314
|
+
]
|
|
315
|
+
.map(([title, value]) => formatSection(title, value))
|
|
316
|
+
.filter(Boolean);
|
|
317
|
+
|
|
318
|
+
if (sections.length === 0) return "";
|
|
319
|
+
const body = sections.join("\n\n");
|
|
320
|
+
if (!hasUsefulMemory(body)) return "";
|
|
321
|
+
// Lead with the single most query-relevant belief as a banner, so the decisive fact is the
|
|
322
|
+
// FIRST thing the agent reads — not buried mid-block where it gets skipped and re-derived.
|
|
323
|
+
const banner = context.headline
|
|
324
|
+
? `\n⚠ MOST RELEVANT — this is already in memory; use it and do NOT re-run/re-derive to rediscover it:\n ${context.headline}\n`
|
|
325
|
+
: "";
|
|
326
|
+
return `${PEON_FIRST_DIRECTIVE}${banner}\nPeon Relevant Memory\n${body.slice(0, 4500)}\n`;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function formatStartupContext(context) {
|
|
330
|
+
const sections = [
|
|
331
|
+
["Peon Global Brain (applies to every project)", context.global],
|
|
332
|
+
["Summary", context.summary],
|
|
333
|
+
["Memory Records", context.memories],
|
|
334
|
+
["Decisions", context.decisions],
|
|
335
|
+
["Preferences", context.preferences],
|
|
336
|
+
["Open Questions", context.openQuestions],
|
|
337
|
+
["Artifacts", context.artifacts],
|
|
338
|
+
["Recent Timeline", context.timeline]
|
|
339
|
+
]
|
|
340
|
+
.map(([title, value]) => formatSection(title, value))
|
|
341
|
+
.filter(Boolean);
|
|
342
|
+
|
|
343
|
+
if (sections.length === 0) return "";
|
|
344
|
+
const body = sections.join("\n\n");
|
|
345
|
+
if (!hasUsefulMemory(body)) return "";
|
|
346
|
+
return `${PEON_FIRST_DIRECTIVE}\nPeon Context\n${body.slice(0, 6000)}\n`;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function formatSection(title, value) {
|
|
350
|
+
const text = String(value || "").trim();
|
|
351
|
+
if (!text || text === `# ${title}`) return "";
|
|
352
|
+
if (text.includes("No recorded memory yet.")) return "";
|
|
353
|
+
const cleaned = text
|
|
354
|
+
.split(/\r?\n/)
|
|
355
|
+
.filter((line) => !line.startsWith("# "))
|
|
356
|
+
.slice(0, 24)
|
|
357
|
+
.join("\n")
|
|
358
|
+
.trim();
|
|
359
|
+
return cleaned ? `## ${title}\n${cleaned}` : "";
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function hasUsefulMemory(text) {
|
|
363
|
+
return /AI Summary|Recent Memory|^- |XO|decision|preference|artifact|commit|implemented|built/im.test(text);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
async function readStdinJson() {
|
|
367
|
+
let raw = "";
|
|
368
|
+
for await (const chunk of process.stdin) raw += chunk;
|
|
369
|
+
if (!raw.trim()) return {};
|
|
370
|
+
try {
|
|
371
|
+
return JSON.parse(raw);
|
|
372
|
+
} catch (error) {
|
|
373
|
+
throw new Error(`invalid_json: ${error instanceof Error ? error.message : String(error)}`);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
async function readSession(externalSessionId) {
|
|
378
|
+
const path = sessionPath(externalSessionId);
|
|
379
|
+
try {
|
|
380
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
381
|
+
} catch {
|
|
382
|
+
return undefined;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
async function writeSession(externalSessionId, session) {
|
|
387
|
+
await mkdir(stateDir, { recursive: true });
|
|
388
|
+
await writeFile(sessionPath(externalSessionId), JSON.stringify(session, null, 2), "utf8");
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
async function removeSession(externalSessionId) {
|
|
392
|
+
await rm(sessionPath(externalSessionId), { force: true }).catch(() => undefined);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function sessionPath(externalSessionId) {
|
|
396
|
+
return join(stateDir, `${safeName(externalSessionId)}.json`);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
function safeName(value) {
|
|
400
|
+
return String(value).replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
const TOKEN_AB_LOG = join(homedir(), "Library", "Application Support", "Peon", "token-ab-log.jsonl");
|
|
404
|
+
|
|
405
|
+
const TOKEN_AB_LOGGED_SESSIONS = join(homedir(), "Library", "Application Support", "Peon", "token-ab-sessions.json");
|
|
406
|
+
|
|
407
|
+
async function recordTokenUsage({ projectPath, externalSessionId, transcriptPath }) {
|
|
408
|
+
try {
|
|
409
|
+
// Only log once per session — Stop fires on every response turn, not just session end.
|
|
410
|
+
const logged = JSON.parse(await readFile(TOKEN_AB_LOGGED_SESSIONS, "utf8").catch(() => "[]"));
|
|
411
|
+
if (logged.includes(externalSessionId)) return;
|
|
412
|
+
|
|
413
|
+
const totals = { input: 0, output: 0, cacheRead: 0, cacheCreate: 0, model: "unknown" };
|
|
414
|
+
const rl = createInterface({ input: fsCreateReadStream(transcriptPath), crlfDelay: Infinity });
|
|
415
|
+
for await (const line of rl) {
|
|
416
|
+
if (!line.trim()) continue;
|
|
417
|
+
try {
|
|
418
|
+
const obj = JSON.parse(line);
|
|
419
|
+
const usage = obj?.message?.usage;
|
|
420
|
+
if (!usage) continue;
|
|
421
|
+
totals.input += usage.input_tokens || 0;
|
|
422
|
+
totals.output += usage.output_tokens || 0;
|
|
423
|
+
totals.cacheRead += usage.cache_read_input_tokens || 0;
|
|
424
|
+
totals.cacheCreate += usage.cache_creation_input_tokens || 0;
|
|
425
|
+
if (obj?.message?.model) totals.model = obj.message.model;
|
|
426
|
+
} catch { /* skip malformed lines */ }
|
|
427
|
+
}
|
|
428
|
+
const peonEnabled = !/^(1|true|yes|on)$/i.test(process.env.PEON_DISABLED || "");
|
|
429
|
+
const record = {
|
|
430
|
+
ts: new Date().toISOString(),
|
|
431
|
+
projectPath,
|
|
432
|
+
sessionId: externalSessionId,
|
|
433
|
+
peonEnabled,
|
|
434
|
+
model: totals.model,
|
|
435
|
+
inputTokens: totals.input,
|
|
436
|
+
outputTokens: totals.output,
|
|
437
|
+
cacheReadTokens: totals.cacheRead,
|
|
438
|
+
cacheCreateTokens: totals.cacheCreate,
|
|
439
|
+
totalTokens: totals.input + totals.output,
|
|
440
|
+
};
|
|
441
|
+
const dir = join(homedir(), "Library", "Application Support", "Peon");
|
|
442
|
+
await mkdir(dir, { recursive: true });
|
|
443
|
+
await appendFile(TOKEN_AB_LOG, JSON.stringify(record) + "\n", "utf8");
|
|
444
|
+
// Mark session as logged (keep last 500 to avoid unbounded growth)
|
|
445
|
+
logged.push(externalSessionId);
|
|
446
|
+
await writeFile(TOKEN_AB_LOGGED_SESSIONS, JSON.stringify(logged.slice(-500)), "utf8");
|
|
447
|
+
} catch { /* non-fatal */ }
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
async function appendLocalError(error) {
|
|
451
|
+
await mkdir(stateDir, { recursive: true });
|
|
452
|
+
await appendFile(join(stateDir, "errors.jsonl"), `${JSON.stringify(error)}\n`, "utf8");
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
function normalizeEventName(value) {
|
|
456
|
+
const text = extractText(value).trim();
|
|
457
|
+
const compact = text.replace(/[^a-zA-Z]/g, "").toLowerCase();
|
|
458
|
+
if (compact === "subagentstart" || compact === "subagentstarted") return "SubagentStart";
|
|
459
|
+
if (compact === "subagentstop" || compact === "subagentend" || compact === "subagentfinished") return "SubagentStop";
|
|
460
|
+
if (compact === "sessionstart" || compact === "start") return "SessionStart";
|
|
461
|
+
if (compact === "userpromptsubmit" || compact === "promptsubmit" || compact === "userprompt") return "UserPromptSubmit";
|
|
462
|
+
if (compact === "userpromptexpansion" || compact === "promptexpansion") return "UserPromptExpansion";
|
|
463
|
+
if (compact === "pretooluse" || compact === "pretool") return "PreToolUse";
|
|
464
|
+
if (compact === "posttooluse" || compact === "tooluse" || compact === "toolresult") return "PostToolUse";
|
|
465
|
+
if (compact === "posttoolusefailure" || compact === "toolusefailure" || compact === "toolerror") return "PostToolUseFailure";
|
|
466
|
+
if (compact === "stop" || compact === "sessionend" || compact === "end") return text === "SessionEnd" ? "SessionEnd" : "Stop";
|
|
467
|
+
return text;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function extractAssistantSummary(input) {
|
|
471
|
+
return extractText(
|
|
472
|
+
readFirst(input, [
|
|
473
|
+
"last_assistant_message",
|
|
474
|
+
"lastAssistantMessage",
|
|
475
|
+
"assistant_summary",
|
|
476
|
+
"assistantSummary",
|
|
477
|
+
"response.message.content",
|
|
478
|
+
"response.message",
|
|
479
|
+
"response",
|
|
480
|
+
"message.content",
|
|
481
|
+
"message",
|
|
482
|
+
"output"
|
|
483
|
+
])
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
function readText(input, paths) {
|
|
488
|
+
return extractText(readFirst(input, paths)).trim();
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
function readFirst(input, paths) {
|
|
492
|
+
for (const path of paths) {
|
|
493
|
+
const value = readPath(input, path);
|
|
494
|
+
if (value === undefined || value === null) continue;
|
|
495
|
+
if (typeof value === "object") return value;
|
|
496
|
+
if (extractText(value).trim()) return value;
|
|
497
|
+
}
|
|
498
|
+
return undefined;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function readPath(input, path) {
|
|
502
|
+
return path.split(".").reduce((value, key) => {
|
|
503
|
+
if (value && typeof value === "object" && key in value) return value[key];
|
|
504
|
+
return undefined;
|
|
505
|
+
}, input);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function extractText(value) {
|
|
509
|
+
if (value === undefined || value === null) return "";
|
|
510
|
+
if (typeof value === "string") return value;
|
|
511
|
+
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
512
|
+
if (Array.isArray(value)) return value.map(extractText).filter(Boolean).join("\n");
|
|
513
|
+
if (typeof value === "object") {
|
|
514
|
+
if (typeof value.text === "string") return value.text;
|
|
515
|
+
if (typeof value.content === "string") return value.content;
|
|
516
|
+
if (Array.isArray(value.content)) return extractText(value.content);
|
|
517
|
+
if (typeof value.message === "string" || Array.isArray(value.message) || typeof value.message === "object") {
|
|
518
|
+
return extractText(value.message);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
return "";
|
|
522
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Labeled query-driven retrieval eval for Peon — measures the entity-graph's real lift with
|
|
3
|
+
// LLM-JUDGED relevance (not the shared-entity proxy). Two phases:
|
|
4
|
+
//
|
|
5
|
+
// build: node scripts/eval-retrieval-labeled.mjs build "<projectPath>" [N=20] [qrelsOut]
|
|
6
|
+
// Samples N active beliefs, turns each into a natural question (without reusing its
|
|
7
|
+
// distinctive words), pools graph-off + graph-on candidates, and has an LLM judge which
|
|
8
|
+
// pool members are relevant. Writes qrels JSON.
|
|
9
|
+
//
|
|
10
|
+
// run: node scripts/eval-retrieval-labeled.mjs run "<projectPath>" [qrelsFile] [K=10]
|
|
11
|
+
// For each labeled question, retrieves top-K with the graph OFF vs ON and reports
|
|
12
|
+
// Recall@K / MRR / nDCG@K for each, plus the lift.
|
|
13
|
+
//
|
|
14
|
+
// Model: PEON_EVAL_MODEL (default = config processingModel). OpenRouter only (never Wulver).
|
|
15
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { PeonMemoryStore } from "../dist/memory-store.js";
|
|
18
|
+
import { loadPeonConfig } from "../dist/config.js";
|
|
19
|
+
import { scoreQuery, aggregate } from "../dist/eval-metrics.js";
|
|
20
|
+
import { gitSha, fileHash, brainFingerprint, ledgerPath, readLedger, findBaseline, appendRow } from "./lib/eval-ledger.mjs";
|
|
21
|
+
|
|
22
|
+
const [, , mode, projectPath, arg3, arg4] = process.argv;
|
|
23
|
+
const cfg = loadPeonConfig();
|
|
24
|
+
const KEY = cfg.openRouterApiKey;
|
|
25
|
+
const MODEL = process.env.PEON_EVAL_MODEL ?? cfg.processingModel;
|
|
26
|
+
if (!mode || !projectPath) { console.error("usage: build|run \"<projectPath>\" ..."); process.exit(1); }
|
|
27
|
+
|
|
28
|
+
async function chat(messages, temperature = 0) {
|
|
29
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
30
|
+
try {
|
|
31
|
+
const r = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
|
32
|
+
method: "POST",
|
|
33
|
+
headers: { Authorization: `Bearer ${KEY}`, "Content-Type": "application/json" },
|
|
34
|
+
body: JSON.stringify({ model: MODEL, temperature, messages })
|
|
35
|
+
});
|
|
36
|
+
if (!r.ok) throw new Error(String(r.status));
|
|
37
|
+
const j = await r.json();
|
|
38
|
+
return (j.choices?.[0]?.message?.content ?? "").trim();
|
|
39
|
+
} catch { await new Promise((res) => setTimeout(res, 800 * (attempt + 1))); }
|
|
40
|
+
}
|
|
41
|
+
return "";
|
|
42
|
+
}
|
|
43
|
+
const parseNums = (s) => { const m = s.match(/\[[\s\S]*?\]/); if (!m) return []; try { return JSON.parse(m[0]).map(Number).filter(Number.isInteger); } catch { return []; } };
|
|
44
|
+
const qrelsPath = (out) => out ?? join(projectPath, ".peon", "evaluation", "qrels.json");
|
|
45
|
+
|
|
46
|
+
async function build() {
|
|
47
|
+
const N = Number.parseInt(arg3 ?? "20", 10);
|
|
48
|
+
const out = qrelsPath(arg4);
|
|
49
|
+
const store = await PeonMemoryStore.open({ projectPath });
|
|
50
|
+
const active = (await store.listMemoryRecords()).filter((r) => r.status === "active" && r.content.trim().length > 20);
|
|
51
|
+
// Deterministic spread: prefer beliefs carrying domain entities (associative recall is testable there).
|
|
52
|
+
const domainish = active.filter((r) => (r.entities ?? []).some((e) => /[a-z].* .*|[a-z][A-Z]/.test(e) || (!e.includes("/") && e.length > 2)));
|
|
53
|
+
const pickFrom = domainish.length >= N ? domainish : active;
|
|
54
|
+
const step = Math.max(1, Math.floor(pickFrom.length / N));
|
|
55
|
+
const cues = pickFrom.filter((_, i) => i % step === 0).slice(0, N);
|
|
56
|
+
|
|
57
|
+
const qrels = [];
|
|
58
|
+
let i = 0;
|
|
59
|
+
for (const cue of cues) {
|
|
60
|
+
const question = await chat([
|
|
61
|
+
{ role: "system", content: "Turn the note into ONE natural question a user might later ask that this note answers. Do NOT reuse the note's distinctive proper nouns/numbers verbatim where a natural paraphrase exists. Output only the question." },
|
|
62
|
+
{ role: "user", content: cue.content }
|
|
63
|
+
]);
|
|
64
|
+
if (!question) continue;
|
|
65
|
+
const off = await store.rankRecords(question, { limit: 15 });
|
|
66
|
+
const on = await store.rankRecords(question, { limit: 15, expandGraph: true });
|
|
67
|
+
const poolMap = new Map();
|
|
68
|
+
for (const r of [...off, ...on, { record: cue }]) poolMap.set(r.record.id, r.record);
|
|
69
|
+
const pool = [...poolMap.values()];
|
|
70
|
+
const numbered = pool.map((r, n) => `${n + 1}. ${r.content.replace(/\s+/g, " ").slice(0, 160)}`).join("\n");
|
|
71
|
+
const verdict = await chat([
|
|
72
|
+
{ role: "system", content: "Given a question and numbered notes, return ONLY a JSON array of the numbers of notes that are RELEVANT to answering it (directly or as useful related context). No prose." },
|
|
73
|
+
{ role: "user", content: `Question: ${question}\n\nNotes:\n${numbered}\n\nRelevant numbers:` }
|
|
74
|
+
]);
|
|
75
|
+
const relevantIds = [...new Set([cue.id, ...parseNums(verdict).map((nn) => pool[nn - 1]?.id).filter(Boolean)])];
|
|
76
|
+
qrels.push({ question, cueId: cue.id, relevantIds });
|
|
77
|
+
process.stderr.write(`\r labeled ${++i}/${cues.length} (${relevantIds.length} relevant)`);
|
|
78
|
+
}
|
|
79
|
+
mkdirSync(join(projectPath, ".peon", "evaluation"), { recursive: true });
|
|
80
|
+
writeFileSync(out, JSON.stringify(qrels, null, 2));
|
|
81
|
+
console.log(`\nwrote ${qrels.length} labeled queries -> ${out}`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function run() {
|
|
85
|
+
const file = arg3 && existsSync(arg3) ? arg3 : qrelsPath();
|
|
86
|
+
const K = Number.parseInt(arg4 ?? "10", 10);
|
|
87
|
+
const qrels = JSON.parse(readFileSync(file, "utf8"));
|
|
88
|
+
const store = await PeonMemoryStore.open({ projectPath });
|
|
89
|
+
const off = [], on = [];
|
|
90
|
+
for (const q of qrels) {
|
|
91
|
+
const rel = new Set(q.relevantIds);
|
|
92
|
+
const idsOff = (await store.rankRecords(q.question, { limit: K })).map((r) => r.record.id);
|
|
93
|
+
const idsOn = (await store.rankRecords(q.question, { limit: K, expandGraph: true })).map((r) => r.record.id);
|
|
94
|
+
off.push(scoreQuery(idsOff, rel, K));
|
|
95
|
+
on.push(scoreQuery(idsOn, rel, K));
|
|
96
|
+
}
|
|
97
|
+
const a = aggregate(off), b = aggregate(on);
|
|
98
|
+
const pct = (x) => (100 * x).toFixed(1) + "%";
|
|
99
|
+
console.log(`Labeled retrieval eval — ${qrels.length} queries, K=${K}, model=${MODEL}`);
|
|
100
|
+
console.log(` Recall@${K} MRR nDCG@${K}`);
|
|
101
|
+
console.log(` graph OFF : ${pct(a.recallAtK).padStart(7)} ${a.mrr.toFixed(3)} ${pct(a.ndcgAtK).padStart(7)}`);
|
|
102
|
+
console.log(` graph ON : ${pct(b.recallAtK).padStart(7)} ${b.mrr.toFixed(3)} ${pct(b.ndcgAtK).padStart(7)}`);
|
|
103
|
+
console.log(` lift : ${pct(b.recallAtK - a.recallAtK).padStart(7)} ${(b.mrr - a.mrr >= 0 ? "+" : "") + (b.mrr - a.mrr).toFixed(3)} ${pct(b.ndcgAtK - a.ndcgAtK).padStart(7)}`);
|
|
104
|
+
|
|
105
|
+
// Ledger: append this run + diff against the last comparable baseline, so the graph-is-dead
|
|
106
|
+
// result (and any future retrieval change) stays continuously guarded rather than a one-time memory.
|
|
107
|
+
const row = {
|
|
108
|
+
ts: new Date().toISOString(),
|
|
109
|
+
gitSha: gitSha(),
|
|
110
|
+
kind: "labeled-retrieval",
|
|
111
|
+
projectPath,
|
|
112
|
+
k: K,
|
|
113
|
+
model: MODEL,
|
|
114
|
+
queries: qrels.length,
|
|
115
|
+
qrelsHash: fileHash(file),
|
|
116
|
+
brain: brainFingerprint(projectPath),
|
|
117
|
+
metrics: { recallOff: a.recallAtK, mrrOff: a.mrr, ndcgOff: a.ndcgAtK, recallOn: b.recallAtK, mrrOn: b.mrr, ndcgOn: b.ndcgAtK }
|
|
118
|
+
};
|
|
119
|
+
const lp = ledgerPath();
|
|
120
|
+
const base = findBaseline(readLedger(lp), row);
|
|
121
|
+
appendRow(lp, row);
|
|
122
|
+
console.log(`\nledger : +1 row → ${lp} (git ${row.gitSha}, brain ${row.brain.records} recs/${row.brain.hash}, qrels ${row.qrelsHash})`);
|
|
123
|
+
if (!base) {
|
|
124
|
+
console.log(" Δ : no comparable baseline yet — this run IS the baseline.");
|
|
125
|
+
} else {
|
|
126
|
+
const dp = (x) => `${x >= 0 ? "+" : ""}${(100 * x).toFixed(1)}%`;
|
|
127
|
+
const dm = (x) => `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;
|
|
128
|
+
const m = base.row.metrics;
|
|
129
|
+
const tag = base.sameBrain ? "vs last run, SAME brain (trustworthy A/B)" : "vs last run — ⚠ brain changed, informational only";
|
|
130
|
+
console.log(` Δ ${tag}:`);
|
|
131
|
+
console.log(` Recall@${K} off ${dp(a.recallAtK - m.recallOff)} / on ${dp(b.recallAtK - m.recallOn)} MRR off ${dm(a.mrr - m.mrrOff)} / on ${dm(b.mrr - m.mrrOn)}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
await (mode === "build" ? build() : run());
|