peon-mem 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +301 -0
- package/bin/peon-mem.mjs +273 -0
- package/dist/brain.d.ts +72 -0
- package/dist/brain.js +224 -0
- package/dist/compression.d.ts +9 -0
- package/dist/compression.js +37 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.js +99 -0
- package/dist/daemon-cli.d.ts +2 -0
- package/dist/daemon-cli.js +54 -0
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +1078 -0
- package/dist/embedding-store.d.ts +43 -0
- package/dist/embedding-store.js +169 -0
- package/dist/embeddings.d.ts +93 -0
- package/dist/embeddings.js +345 -0
- package/dist/entities.d.ts +61 -0
- package/dist/entities.js +191 -0
- package/dist/entity-extraction.d.ts +33 -0
- package/dist/entity-extraction.js +75 -0
- package/dist/eval-metrics.d.ts +27 -0
- package/dist/eval-metrics.js +50 -0
- package/dist/evaluation.d.ts +58 -0
- package/dist/evaluation.js +244 -0
- package/dist/global-extraction.d.ts +15 -0
- package/dist/global-extraction.js +61 -0
- package/dist/global-memory.d.ts +43 -0
- package/dist/global-memory.js +306 -0
- package/dist/global-promotion.d.ts +25 -0
- package/dist/global-promotion.js +29 -0
- package/dist/hyde.d.ts +31 -0
- package/dist/hyde.js +46 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +246 -0
- package/dist/injection.d.ts +38 -0
- package/dist/injection.js +133 -0
- package/dist/logger.d.ts +17 -0
- package/dist/logger.js +63 -0
- package/dist/memory-mutations.d.ts +24 -0
- package/dist/memory-mutations.js +57 -0
- package/dist/memory-store.d.ts +194 -0
- package/dist/memory-store.js +1205 -0
- package/dist/monitor.d.ts +13 -0
- package/dist/monitor.js +977 -0
- package/dist/overview.d.ts +73 -0
- package/dist/overview.js +104 -0
- package/dist/processor.d.ts +90 -0
- package/dist/processor.js +450 -0
- package/dist/quality.d.ts +86 -0
- package/dist/quality.js +338 -0
- package/dist/recuration.d.ts +13 -0
- package/dist/recuration.js +65 -0
- package/dist/reranker.d.ts +34 -0
- package/dist/reranker.js +89 -0
- package/dist/retrieval.d.ts +106 -0
- package/dist/retrieval.js +392 -0
- package/dist/session-index.d.ts +34 -0
- package/dist/session-index.js +87 -0
- package/dist/temporal.d.ts +20 -0
- package/dist/temporal.js +62 -0
- package/dist/token-ab-monitor.d.ts +1 -0
- package/dist/token-ab-monitor.js +7 -0
- package/dist/tools.d.ts +232 -0
- package/dist/tools.js +546 -0
- package/dist/types.d.ts +169 -0
- package/dist/types.js +1 -0
- package/docs/assets/neural-universe.png +0 -0
- package/package.json +57 -0
- package/scripts/claude-peon-hook.mjs +522 -0
- package/scripts/codex-peon-hook.mjs +4 -0
- package/scripts/eval-retrieval-labeled.mjs +135 -0
- package/scripts/eval-retrieval.mjs +96 -0
- package/scripts/evaluate-peon.mjs +47 -0
- package/scripts/install-peon-stl.mjs +82 -0
- package/scripts/install-peon.mjs +318 -0
- package/scripts/lib/eval-ledger.mjs +104 -0
- package/scripts/lib/stl-classify.mjs +44 -0
- package/scripts/longmemeval-eval.mjs +144 -0
- package/scripts/peon-report.mjs +155 -0
- package/scripts/peon-stl.mjs +506 -0
- package/scripts/token-ab-monitor.html +235 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
export function computeEvaluationReport(input) {
|
|
4
|
+
const expected = normalizeExpected(input.expectedMemories);
|
|
5
|
+
const observed = [
|
|
6
|
+
...extractObservedItems(input.retrievedText, "retrieved"),
|
|
7
|
+
...extractObservedItems(input.injectedText, "injected")
|
|
8
|
+
];
|
|
9
|
+
const matches = matchExpectedToObserved(expected, observed);
|
|
10
|
+
const matchedExpectedIds = new Set(matches.map((match) => match.expectedId));
|
|
11
|
+
const matchedObservedKeys = new Set(matches.map((match) => observedKey(match.observedSource, match.observedContent)));
|
|
12
|
+
return {
|
|
13
|
+
expectedCount: expected.length,
|
|
14
|
+
observedItemCount: observed.length,
|
|
15
|
+
matchedExpectedCount: matchedExpectedIds.size,
|
|
16
|
+
matchedObservedItemCount: matchedObservedKeys.size,
|
|
17
|
+
recall: ratio(matchedExpectedIds.size, expected.length),
|
|
18
|
+
coverage: ratio(matchedObservedKeys.size, observed.length),
|
|
19
|
+
missingExpectedItems: expected
|
|
20
|
+
.filter((item) => !matchedExpectedIds.has(item.id))
|
|
21
|
+
.map((item) => ({ id: item.id, content: item.content })),
|
|
22
|
+
unexpectedNoisyItems: observed
|
|
23
|
+
.filter((item) => !matchedObservedKeys.has(observedKey(item.source, item.content)))
|
|
24
|
+
.map((item) => ({ source: item.source, content: item.content })),
|
|
25
|
+
matches,
|
|
26
|
+
costSummary: summarizeCost(input.processingJobs ?? [])
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export async function evaluatePeonProject(input) {
|
|
30
|
+
const peonPath = join(input.projectPath, input.memoryDirName ?? ".peon");
|
|
31
|
+
const expectedMemories = input.expectedMemories ?? (await readExpectedMemories(peonPath));
|
|
32
|
+
const retrievedText = await readRetrievedProjectText(peonPath);
|
|
33
|
+
const injectedText = await readOptionalText(join(peonPath, "brain", "injection-preview.md"));
|
|
34
|
+
const processingJobs = await readProcessingJobs(peonPath);
|
|
35
|
+
return computeEvaluationReport({
|
|
36
|
+
expectedMemories,
|
|
37
|
+
retrievedText,
|
|
38
|
+
injectedText,
|
|
39
|
+
processingJobs
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
function normalizeExpected(items) {
|
|
43
|
+
return items
|
|
44
|
+
.map((item, index) => {
|
|
45
|
+
const content = typeof item === "string" ? item : item.content;
|
|
46
|
+
const id = typeof item === "string" ? `expected-${index + 1}` : item.id ?? `expected-${index + 1}`;
|
|
47
|
+
return {
|
|
48
|
+
id,
|
|
49
|
+
content: content.trim(),
|
|
50
|
+
tokens: tokenize(content),
|
|
51
|
+
normalized: normalizeText(content)
|
|
52
|
+
};
|
|
53
|
+
})
|
|
54
|
+
.filter((item) => item.content.length > 0);
|
|
55
|
+
}
|
|
56
|
+
function extractObservedItems(input, source) {
|
|
57
|
+
const chunks = Array.isArray(input) ? input : input ? [input] : [];
|
|
58
|
+
return chunks.flatMap((chunk) => chunk
|
|
59
|
+
.split(/\r?\n/)
|
|
60
|
+
.map(cleanObservedLine)
|
|
61
|
+
.filter(isMeaningfulObservedLine)
|
|
62
|
+
.map((content) => ({
|
|
63
|
+
source,
|
|
64
|
+
content,
|
|
65
|
+
tokens: tokenize(content),
|
|
66
|
+
normalized: normalizeText(content)
|
|
67
|
+
})));
|
|
68
|
+
}
|
|
69
|
+
function cleanObservedLine(line) {
|
|
70
|
+
return line
|
|
71
|
+
.trim()
|
|
72
|
+
.replace(/^[-*]\s+/, "")
|
|
73
|
+
.replace(/^\d+[.)]\s+/, "")
|
|
74
|
+
.trim();
|
|
75
|
+
}
|
|
76
|
+
function isMeaningfulObservedLine(line) {
|
|
77
|
+
if (!line)
|
|
78
|
+
return false;
|
|
79
|
+
if (/^#+\s*/.test(line))
|
|
80
|
+
return false;
|
|
81
|
+
if (/^[A-Z][A-Za-z ]{1,32}:?$/.test(line))
|
|
82
|
+
return false;
|
|
83
|
+
return tokenize(line).length > 0;
|
|
84
|
+
}
|
|
85
|
+
function matchExpectedToObserved(expected, observed) {
|
|
86
|
+
return expected.flatMap((item) => {
|
|
87
|
+
const best = observed
|
|
88
|
+
.map((candidate) => ({ candidate, score: matchScore(item, candidate) }))
|
|
89
|
+
.filter((candidate) => candidate.score >= 0.6)
|
|
90
|
+
.sort((left, right) => right.score - left.score || sourceRank(left.candidate.source) - sourceRank(right.candidate.source) || left.candidate.content.localeCompare(right.candidate.content))[0];
|
|
91
|
+
return best
|
|
92
|
+
? [
|
|
93
|
+
{
|
|
94
|
+
expectedId: item.id,
|
|
95
|
+
expectedContent: item.content,
|
|
96
|
+
observedSource: best.candidate.source,
|
|
97
|
+
observedContent: best.candidate.content,
|
|
98
|
+
score: round4(best.score)
|
|
99
|
+
}
|
|
100
|
+
]
|
|
101
|
+
: [];
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function matchScore(expected, observed) {
|
|
105
|
+
if (!expected.tokens.length || !observed.tokens.length)
|
|
106
|
+
return 0;
|
|
107
|
+
if (observed.normalized.includes(expected.normalized) || expected.normalized.includes(observed.normalized))
|
|
108
|
+
return 1;
|
|
109
|
+
const observedTokens = new Set(observed.tokens);
|
|
110
|
+
const hits = expected.tokens.filter((token) => observedTokens.has(token)).length;
|
|
111
|
+
return hits / expected.tokens.length;
|
|
112
|
+
}
|
|
113
|
+
function summarizeCost(jobs) {
|
|
114
|
+
const summary = {
|
|
115
|
+
jobCount: jobs.length,
|
|
116
|
+
processedJobs: 0,
|
|
117
|
+
skippedJobs: 0,
|
|
118
|
+
failedJobs: 0,
|
|
119
|
+
totalEstimatedTokens: 0,
|
|
120
|
+
byModel: {}
|
|
121
|
+
};
|
|
122
|
+
for (const job of jobs) {
|
|
123
|
+
const status = job.status ?? "unknown";
|
|
124
|
+
const tokens = Number.isFinite(job.estimatedTokens) ? Math.max(0, Math.trunc(job.estimatedTokens ?? 0)) : 0;
|
|
125
|
+
const model = job.model || "unknown";
|
|
126
|
+
if (status === "processed")
|
|
127
|
+
summary.processedJobs += 1;
|
|
128
|
+
if (status === "skipped")
|
|
129
|
+
summary.skippedJobs += 1;
|
|
130
|
+
if (status === "failed")
|
|
131
|
+
summary.failedJobs += 1;
|
|
132
|
+
summary.totalEstimatedTokens += tokens;
|
|
133
|
+
summary.byModel[model] ??= { jobCount: 0, estimatedTokens: 0 };
|
|
134
|
+
summary.byModel[model].jobCount += 1;
|
|
135
|
+
summary.byModel[model].estimatedTokens += tokens;
|
|
136
|
+
}
|
|
137
|
+
return summary;
|
|
138
|
+
}
|
|
139
|
+
async function readExpectedMemories(peonPath) {
|
|
140
|
+
const candidates = [
|
|
141
|
+
join(peonPath, "evaluation", "expected-memories.json"),
|
|
142
|
+
join(peonPath, "expected-memories.json")
|
|
143
|
+
];
|
|
144
|
+
for (const candidate of candidates) {
|
|
145
|
+
const text = await readOptionalText(candidate);
|
|
146
|
+
if (!text.trim())
|
|
147
|
+
continue;
|
|
148
|
+
const parsed = JSON.parse(text);
|
|
149
|
+
if (!Array.isArray(parsed))
|
|
150
|
+
throw new Error(`${candidate} must contain a JSON array.`);
|
|
151
|
+
return parsed.flatMap((item) => {
|
|
152
|
+
if (typeof item === "string")
|
|
153
|
+
return [item];
|
|
154
|
+
if (isObject(item) && typeof item.content === "string") {
|
|
155
|
+
return [{ id: typeof item.id === "string" ? item.id : undefined, content: item.content }];
|
|
156
|
+
}
|
|
157
|
+
return [];
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
return [];
|
|
161
|
+
}
|
|
162
|
+
async function readRetrievedProjectText(peonPath) {
|
|
163
|
+
const brainPath = join(peonPath, "brain");
|
|
164
|
+
const markdownFiles = [
|
|
165
|
+
"project-summary.md",
|
|
166
|
+
"decisions.md",
|
|
167
|
+
"preferences.md",
|
|
168
|
+
"open-questions.md",
|
|
169
|
+
"artifacts.md",
|
|
170
|
+
"timeline.md"
|
|
171
|
+
];
|
|
172
|
+
const markdown = await Promise.all(markdownFiles.map((file) => readOptionalText(join(brainPath, file))));
|
|
173
|
+
const records = await readMemoryRecordContents(join(brainPath, "memories.jsonl"));
|
|
174
|
+
return [...markdown, records.join("\n")].filter((text) => text.trim().length > 0);
|
|
175
|
+
}
|
|
176
|
+
async function readMemoryRecordContents(path) {
|
|
177
|
+
const text = await readOptionalText(path);
|
|
178
|
+
return text
|
|
179
|
+
.split(/\r?\n/)
|
|
180
|
+
.filter(Boolean)
|
|
181
|
+
.flatMap((line) => {
|
|
182
|
+
const parsed = JSON.parse(line);
|
|
183
|
+
return isObject(parsed) && typeof parsed.content === "string" ? [parsed.content] : [];
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
async function readProcessingJobs(peonPath) {
|
|
187
|
+
const jobsPath = join(peonPath, "brain", "processing-jobs.json");
|
|
188
|
+
const jobsText = await readOptionalText(jobsPath);
|
|
189
|
+
if (jobsText.trim()) {
|
|
190
|
+
const parsed = JSON.parse(jobsText);
|
|
191
|
+
if (Array.isArray(parsed))
|
|
192
|
+
return parsed.filter(isObject).map(toProcessingJob);
|
|
193
|
+
}
|
|
194
|
+
const stateText = await readOptionalText(join(peonPath, "brain", "processing-state.json"));
|
|
195
|
+
if (!stateText.trim())
|
|
196
|
+
return [];
|
|
197
|
+
const state = JSON.parse(stateText);
|
|
198
|
+
if (!isObject(state) || !state.lastStatus)
|
|
199
|
+
return [];
|
|
200
|
+
return [
|
|
201
|
+
toProcessingJob({
|
|
202
|
+
status: state.lastStatus,
|
|
203
|
+
model: state.lastModel,
|
|
204
|
+
reason: state.lastReason,
|
|
205
|
+
estimatedTokens: state.lastEstimatedTokens
|
|
206
|
+
})
|
|
207
|
+
];
|
|
208
|
+
}
|
|
209
|
+
function toProcessingJob(value) {
|
|
210
|
+
return {
|
|
211
|
+
status: typeof value.status === "string" ? value.status : undefined,
|
|
212
|
+
model: typeof value.model === "string" ? value.model : undefined,
|
|
213
|
+
reason: typeof value.reason === "string" ? value.reason : undefined,
|
|
214
|
+
estimatedTokens: typeof value.estimatedTokens === "number" ? value.estimatedTokens : undefined
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
async function readOptionalText(path) {
|
|
218
|
+
return readFile(path, "utf8").catch((error) => {
|
|
219
|
+
if (isObject(error) && error.code === "ENOENT")
|
|
220
|
+
return "";
|
|
221
|
+
throw error;
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
function tokenize(value) {
|
|
225
|
+
return Array.from(new Set(normalizeText(value).split(" ").filter((token) => token.length > 1)));
|
|
226
|
+
}
|
|
227
|
+
function normalizeText(value) {
|
|
228
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
|
|
229
|
+
}
|
|
230
|
+
function ratio(numerator, denominator) {
|
|
231
|
+
return denominator === 0 ? 1 : round4(numerator / denominator);
|
|
232
|
+
}
|
|
233
|
+
function round4(value) {
|
|
234
|
+
return Math.round(value * 10000) / 10000;
|
|
235
|
+
}
|
|
236
|
+
function sourceRank(source) {
|
|
237
|
+
return source === "retrieved" ? 0 : 1;
|
|
238
|
+
}
|
|
239
|
+
function observedKey(source, content) {
|
|
240
|
+
return `${source}\0${content}`;
|
|
241
|
+
}
|
|
242
|
+
function isObject(value) {
|
|
243
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
244
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { PeonConfig } from "./config.js";
|
|
2
|
+
import type { MemoryRecord } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* The AI-judged path that lets global memory actually build up. A blunt type rule
|
|
5
|
+
* can't tell "the user runs on the NJIT cluster" (global) from "use flash_attention_2
|
|
6
|
+
* to match this paper" (project-local) — both are `preference`s. So we ask the cheap
|
|
7
|
+
* consolidation model to pick out ONLY the cross-cutting beliefs.
|
|
8
|
+
*
|
|
9
|
+
* Returns a function that takes a project's beliefs and yields concise global facts.
|
|
10
|
+
* null when AI is off / no key (global then only grows via explicit promotion).
|
|
11
|
+
*/
|
|
12
|
+
export type GlobalExtractor = (records: readonly MemoryRecord[]) => Promise<string[]>;
|
|
13
|
+
export declare function createGlobalExtractor(config: PeonConfig): GlobalExtractor | null;
|
|
14
|
+
/** Tolerant parse of the model's reply into a clean string list. */
|
|
15
|
+
export declare function parseStringArray(content: string): string[];
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export function createGlobalExtractor(config) {
|
|
2
|
+
if (config.aiMode === "off" || !config.openRouterApiKey)
|
|
3
|
+
return null;
|
|
4
|
+
return async (records) => {
|
|
5
|
+
// Send the highest-signal beliefs only — bounds tokens, focuses the model.
|
|
6
|
+
const candidates = records
|
|
7
|
+
.filter((r) => r.status === "active" && r.type !== "timeline" && r.type !== "open_question")
|
|
8
|
+
.sort((a, b) => b.score.importance - a.score.importance)
|
|
9
|
+
.slice(0, 60);
|
|
10
|
+
if (candidates.length === 0)
|
|
11
|
+
return [];
|
|
12
|
+
const list = candidates.map((r, i) => `${i + 1}. [${r.type}] ${r.content}`).join("\n");
|
|
13
|
+
const system = "You curate a user's GLOBAL memory — facts true across ALL of their projects, independent of which one they work on. " +
|
|
14
|
+
"The beliefs below come from working ON one specific software/research project. " +
|
|
15
|
+
"CRITICAL: that project's OWN internals are NOT global, even when they sound general — exclude its tools, APIs, " +
|
|
16
|
+
"architecture, modules, data model, config flags, UI, code decisions, and bugs. If a fact describes how the project " +
|
|
17
|
+
"being analyzed is built or behaves, DROP it. " +
|
|
18
|
+
"Extract ONLY facts about the USER and their broader environment that would hold in a completely different project: " +
|
|
19
|
+
"their compute hardware and clusters (names, hostnames, GPUs, scratch paths), OS, external accounts/services, " +
|
|
20
|
+
"shell/CLI habits and tools they reuse everywhere (rsync, gh, SLURM commands), and durable personal facts. " +
|
|
21
|
+
"Example KEEP: 'The user runs GPU jobs on the NJIT Wulver cluster via SLURM.' " +
|
|
22
|
+
"Example DROP (project-internal): 'The daemon exposes a /global/extract endpoint.' " +
|
|
23
|
+
"Rewrite each as one self-contained sentence with zero project context. " +
|
|
24
|
+
"Output ONLY a JSON array of strings — no markdown fences, no prose. If nothing qualifies, return []. Max 8 items.";
|
|
25
|
+
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
|
26
|
+
method: "POST",
|
|
27
|
+
headers: { Authorization: `Bearer ${config.openRouterApiKey}`, "Content-Type": "application/json" },
|
|
28
|
+
body: JSON.stringify({
|
|
29
|
+
model: config.processingModel,
|
|
30
|
+
messages: [
|
|
31
|
+
{ role: "system", content: system },
|
|
32
|
+
{ role: "user", content: `Project beliefs:\n${list}\n\nGlobal facts (JSON array):` }
|
|
33
|
+
],
|
|
34
|
+
temperature: 0.1
|
|
35
|
+
})
|
|
36
|
+
});
|
|
37
|
+
if (!response.ok)
|
|
38
|
+
throw new Error(`global extraction failed with ${response.status}`);
|
|
39
|
+
const json = (await response.json());
|
|
40
|
+
return parseStringArray(json.choices?.[0]?.message?.content ?? "");
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/** Tolerant parse of the model's reply into a clean string list. */
|
|
44
|
+
export function parseStringArray(content) {
|
|
45
|
+
const text = content.trim();
|
|
46
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
|
|
47
|
+
const body = fenced ? fenced[1] : text.slice(text.indexOf("["), text.lastIndexOf("]") + 1);
|
|
48
|
+
try {
|
|
49
|
+
const parsed = JSON.parse(body || text);
|
|
50
|
+
if (!Array.isArray(parsed))
|
|
51
|
+
return [];
|
|
52
|
+
return parsed
|
|
53
|
+
.filter((item) => typeof item === "string")
|
|
54
|
+
.map((item) => item.trim())
|
|
55
|
+
.filter((item) => item.length > 0)
|
|
56
|
+
.slice(0, 8);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { MemoryRecord, MemoryRecordInput, MemoryStatus, MemoryType } from "./types.js";
|
|
2
|
+
export interface OpenGlobalMemoryStoreOptions {
|
|
3
|
+
globalDir?: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ListGlobalMemoryOptions {
|
|
6
|
+
query?: string;
|
|
7
|
+
type?: MemoryType;
|
|
8
|
+
status?: MemoryStatus;
|
|
9
|
+
}
|
|
10
|
+
export interface GlobalMemorySource {
|
|
11
|
+
kind?: MemoryRecord["source"]["kind"];
|
|
12
|
+
reason?: string;
|
|
13
|
+
}
|
|
14
|
+
export declare class PeonGlobalMemoryStore {
|
|
15
|
+
private readonly globalDir;
|
|
16
|
+
private static readonly defaultGlobalDir;
|
|
17
|
+
private constructor();
|
|
18
|
+
static defaultDirectory(): string;
|
|
19
|
+
static open(options?: OpenGlobalMemoryStoreOptions): Promise<PeonGlobalMemoryStore>;
|
|
20
|
+
append(input: MemoryRecordInput, source?: GlobalMemorySource): Promise<MemoryRecord>;
|
|
21
|
+
upsert(input: MemoryRecordInput, source?: GlobalMemorySource): Promise<MemoryRecord>;
|
|
22
|
+
list(options?: ListGlobalMemoryOptions): Promise<MemoryRecord[]>;
|
|
23
|
+
search(query: string, options?: Omit<ListGlobalMemoryOptions, "query">): Promise<MemoryRecord[]>;
|
|
24
|
+
/**
|
|
25
|
+
* Curate the GLOBAL brain itself: resolve conflicts and merge duplicates across
|
|
26
|
+
* the shared cross-project memory (global beliefs are the working set here, so
|
|
27
|
+
* they are NOT treated as protected). Snapshots a backup first. LLM compression
|
|
28
|
+
* is opt-in via the summarizer. Returns the actions taken.
|
|
29
|
+
*/
|
|
30
|
+
runBrainPass(options?: {
|
|
31
|
+
summarize?: import("./brain.js").Summarizer;
|
|
32
|
+
}): Promise<import("./brain.js").BrainAction[]>;
|
|
33
|
+
readBrainActions(limit?: number): Promise<Array<{
|
|
34
|
+
at: string;
|
|
35
|
+
actions: import("./brain.js").BrainAction[];
|
|
36
|
+
}>>;
|
|
37
|
+
private snapshotBackup;
|
|
38
|
+
importGlobalRecords(records: MemoryRecord[], source?: GlobalMemorySource): Promise<MemoryRecord[]>;
|
|
39
|
+
private ensureLayout;
|
|
40
|
+
private recordsPath;
|
|
41
|
+
private readRecords;
|
|
42
|
+
private writeRecords;
|
|
43
|
+
}
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
export class PeonGlobalMemoryStore {
|
|
6
|
+
globalDir;
|
|
7
|
+
static defaultGlobalDir = join(homedir(), "Library", "Application Support", "Peon", "global");
|
|
8
|
+
constructor(globalDir) {
|
|
9
|
+
this.globalDir = globalDir;
|
|
10
|
+
}
|
|
11
|
+
static defaultDirectory() {
|
|
12
|
+
return PeonGlobalMemoryStore.defaultGlobalDir;
|
|
13
|
+
}
|
|
14
|
+
static async open(options = {}) {
|
|
15
|
+
const store = new PeonGlobalMemoryStore(options.globalDir ?? PeonGlobalMemoryStore.defaultDirectory());
|
|
16
|
+
await store.ensureLayout();
|
|
17
|
+
return store;
|
|
18
|
+
}
|
|
19
|
+
async append(input, source = {}) {
|
|
20
|
+
const record = createRecord(input, source, { stableId: false });
|
|
21
|
+
await appendFile(this.recordsPath(), `${JSON.stringify(record)}\n`, "utf8");
|
|
22
|
+
return record;
|
|
23
|
+
}
|
|
24
|
+
async upsert(input, source = {}) {
|
|
25
|
+
const records = await this.readRecords();
|
|
26
|
+
const key = memoryKey(input.type, input.content);
|
|
27
|
+
const existingIndex = records.findIndex((record) => memoryKey(record.type, record.content) === key);
|
|
28
|
+
if (existingIndex === -1) {
|
|
29
|
+
const record = createRecord(input, source, { stableId: true });
|
|
30
|
+
records.push(record);
|
|
31
|
+
await this.writeRecords(records);
|
|
32
|
+
return record;
|
|
33
|
+
}
|
|
34
|
+
const existing = records[existingIndex];
|
|
35
|
+
const updated = mergeRecord(existing, input, source);
|
|
36
|
+
records[existingIndex] = updated;
|
|
37
|
+
await this.writeRecords(records);
|
|
38
|
+
return updated;
|
|
39
|
+
}
|
|
40
|
+
async list(options = {}) {
|
|
41
|
+
const terms = searchTerms(options.query);
|
|
42
|
+
return (await this.readRecords())
|
|
43
|
+
.filter((record) => record.scope === "global")
|
|
44
|
+
.filter((record) => (options.type ? record.type === options.type : true))
|
|
45
|
+
.filter((record) => (options.status ? record.status === options.status : true))
|
|
46
|
+
.map((record) => ({ record, relevance: memoryRelevance(record, terms) }))
|
|
47
|
+
.filter(({ relevance }) => terms.length === 0 || relevance > 0)
|
|
48
|
+
.sort((left, right) => compareRankedRecords(left, right))
|
|
49
|
+
.map(({ record }) => record);
|
|
50
|
+
}
|
|
51
|
+
async search(query, options = {}) {
|
|
52
|
+
return this.list({ ...options, query });
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Curate the GLOBAL brain itself: resolve conflicts and merge duplicates across
|
|
56
|
+
* the shared cross-project memory (global beliefs are the working set here, so
|
|
57
|
+
* they are NOT treated as protected). Snapshots a backup first. LLM compression
|
|
58
|
+
* is opt-in via the summarizer. Returns the actions taken.
|
|
59
|
+
*/
|
|
60
|
+
async runBrainPass(options = {}) {
|
|
61
|
+
const { runSleepCycle } = await import("./brain.js");
|
|
62
|
+
const all = await this.readRecords();
|
|
63
|
+
if (all.length === 0)
|
|
64
|
+
return [];
|
|
65
|
+
const now = new Date().toISOString();
|
|
66
|
+
await this.snapshotBackup(all);
|
|
67
|
+
const { records, actions } = await runSleepCycle(all, {
|
|
68
|
+
now,
|
|
69
|
+
summarize: options.summarize,
|
|
70
|
+
protectGlobalScope: false,
|
|
71
|
+
makeSummaryId: (entity) => stableMemoryId("summary", `global:${entity}`)
|
|
72
|
+
});
|
|
73
|
+
if (actions.length === 0)
|
|
74
|
+
return [];
|
|
75
|
+
await this.writeRecords(records);
|
|
76
|
+
await appendFile(join(this.globalDir, "brain-actions.jsonl"), `${JSON.stringify({ at: now, actions })}\n`, "utf8");
|
|
77
|
+
return actions;
|
|
78
|
+
}
|
|
79
|
+
async readBrainActions(limit = 50) {
|
|
80
|
+
const raw = await readFile(join(this.globalDir, "brain-actions.jsonl"), "utf8").catch(() => "");
|
|
81
|
+
const rows = raw.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).flatMap((l) => {
|
|
82
|
+
try {
|
|
83
|
+
return [JSON.parse(l)];
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
return rows.slice(-limit).reverse();
|
|
90
|
+
}
|
|
91
|
+
async snapshotBackup(records) {
|
|
92
|
+
const dir = join(this.globalDir, "backups");
|
|
93
|
+
await mkdir(dir, { recursive: true });
|
|
94
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
95
|
+
await writeFile(join(dir, `memories-${stamp}.jsonl`), records.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8");
|
|
96
|
+
}
|
|
97
|
+
async importGlobalRecords(records, source = {}) {
|
|
98
|
+
const imported = [];
|
|
99
|
+
for (const record of records) {
|
|
100
|
+
if (record.scope !== "global")
|
|
101
|
+
continue;
|
|
102
|
+
imported.push(await this.upsert(memoryRecordToInput(record), {
|
|
103
|
+
kind: source.kind ?? record.source.kind,
|
|
104
|
+
reason: source.reason ?? record.source.reason
|
|
105
|
+
}));
|
|
106
|
+
}
|
|
107
|
+
return imported;
|
|
108
|
+
}
|
|
109
|
+
async ensureLayout() {
|
|
110
|
+
await mkdir(this.globalDir, { recursive: true });
|
|
111
|
+
await readFile(this.recordsPath(), "utf8").catch(async () => {
|
|
112
|
+
await writeFile(this.recordsPath(), "", "utf8");
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
recordsPath() {
|
|
116
|
+
return join(this.globalDir, "memories.jsonl");
|
|
117
|
+
}
|
|
118
|
+
async readRecords() {
|
|
119
|
+
const raw = await readFile(this.recordsPath(), "utf8").catch(() => "");
|
|
120
|
+
return raw
|
|
121
|
+
.split(/\r?\n/)
|
|
122
|
+
.map((line) => line.trim())
|
|
123
|
+
.filter(Boolean)
|
|
124
|
+
.flatMap((line) => {
|
|
125
|
+
try {
|
|
126
|
+
const value = JSON.parse(line);
|
|
127
|
+
return isMemoryRecord(value) ? [value] : [];
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return [];
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
async writeRecords(records) {
|
|
135
|
+
await writeFile(this.recordsPath(), records.map((record) => JSON.stringify(record)).join("\n") + (records.length > 0 ? "\n" : ""), "utf8");
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
function createRecord(input, source, options) {
|
|
139
|
+
const now = new Date().toISOString();
|
|
140
|
+
const content = input.content.trim();
|
|
141
|
+
return {
|
|
142
|
+
id: options.stableId ? stableMemoryId(input.type, content) : `global_${randomUUID()}`,
|
|
143
|
+
type: input.type,
|
|
144
|
+
content,
|
|
145
|
+
normalized: normalizeMemory(content),
|
|
146
|
+
scope: "global",
|
|
147
|
+
status: input.status ?? "active",
|
|
148
|
+
score: scoreMemory(input),
|
|
149
|
+
source: {
|
|
150
|
+
kind: source.kind ?? "manual",
|
|
151
|
+
reason: source.reason
|
|
152
|
+
},
|
|
153
|
+
entities: unique(input.entities ?? []),
|
|
154
|
+
createdAt: now,
|
|
155
|
+
updatedAt: now
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function mergeRecord(existing, input, source) {
|
|
159
|
+
const score = scoreMemory(input);
|
|
160
|
+
return {
|
|
161
|
+
...existing,
|
|
162
|
+
type: input.type,
|
|
163
|
+
content: input.content.trim(),
|
|
164
|
+
normalized: normalizeMemory(input.content),
|
|
165
|
+
scope: "global",
|
|
166
|
+
status: input.status ?? existing.status,
|
|
167
|
+
score: {
|
|
168
|
+
importance: Math.max(existing.score.importance, score.importance),
|
|
169
|
+
confidence: Math.max(existing.score.confidence, score.confidence)
|
|
170
|
+
},
|
|
171
|
+
source: {
|
|
172
|
+
kind: source.kind ?? existing.source.kind,
|
|
173
|
+
reason: source.reason ?? existing.source.reason
|
|
174
|
+
},
|
|
175
|
+
entities: unique([...existing.entities, ...(input.entities ?? [])]),
|
|
176
|
+
updatedAt: new Date().toISOString()
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function memoryRecordToInput(record) {
|
|
180
|
+
return {
|
|
181
|
+
type: record.type,
|
|
182
|
+
content: record.content,
|
|
183
|
+
scope: record.scope,
|
|
184
|
+
importance: record.score.importance,
|
|
185
|
+
confidence: record.score.confidence,
|
|
186
|
+
entities: record.entities,
|
|
187
|
+
status: record.status
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
function scoreMemory(input) {
|
|
191
|
+
const baseImportance = {
|
|
192
|
+
summary: 0.75,
|
|
193
|
+
decision: 0.9,
|
|
194
|
+
preference: 0.75,
|
|
195
|
+
open_question: 0.65,
|
|
196
|
+
artifact: 0.8,
|
|
197
|
+
timeline: 0.55,
|
|
198
|
+
fact: 0.7
|
|
199
|
+
};
|
|
200
|
+
return {
|
|
201
|
+
importance: clamp(input.importance ?? baseImportance[input.type] ?? 0.6),
|
|
202
|
+
confidence: clamp(input.confidence ?? 0.82)
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
function compareRankedRecords(left, right) {
|
|
206
|
+
if (left.relevance !== right.relevance)
|
|
207
|
+
return right.relevance - left.relevance;
|
|
208
|
+
const statusRank = statusWeight(right.record.status) - statusWeight(left.record.status);
|
|
209
|
+
if (statusRank !== 0)
|
|
210
|
+
return statusRank;
|
|
211
|
+
const leftScore = left.record.score.importance + left.record.score.confidence;
|
|
212
|
+
const rightScore = right.record.score.importance + right.record.score.confidence;
|
|
213
|
+
if (leftScore !== rightScore)
|
|
214
|
+
return rightScore - leftScore;
|
|
215
|
+
return right.record.updatedAt.localeCompare(left.record.updatedAt);
|
|
216
|
+
}
|
|
217
|
+
function statusWeight(status) {
|
|
218
|
+
if (status === "active")
|
|
219
|
+
return 2;
|
|
220
|
+
if (status === "conflicted")
|
|
221
|
+
return 1;
|
|
222
|
+
return 0;
|
|
223
|
+
}
|
|
224
|
+
function memoryRelevance(record, terms) {
|
|
225
|
+
if (terms.length === 0)
|
|
226
|
+
return 1;
|
|
227
|
+
const haystack = `${record.type} ${record.content} ${record.entities.join(" ")}`.toLowerCase();
|
|
228
|
+
return terms.filter((term) => haystack.includes(term)).length;
|
|
229
|
+
}
|
|
230
|
+
function searchTerms(query) {
|
|
231
|
+
if (!query)
|
|
232
|
+
return [];
|
|
233
|
+
return unique(query
|
|
234
|
+
.toLowerCase()
|
|
235
|
+
.split(/[^a-z0-9_.-]+/)
|
|
236
|
+
.filter((term) => term.length >= 2)
|
|
237
|
+
.slice(0, 16));
|
|
238
|
+
}
|
|
239
|
+
function memoryKey(type, content) {
|
|
240
|
+
return `${type}:${normalizeMemory(content)}`;
|
|
241
|
+
}
|
|
242
|
+
function normalizeMemory(content) {
|
|
243
|
+
return content
|
|
244
|
+
.toLowerCase()
|
|
245
|
+
.replace(/[`"'.,;:!?()[\]{}]/g, "")
|
|
246
|
+
.replace(/\s+/g, " ")
|
|
247
|
+
.trim();
|
|
248
|
+
}
|
|
249
|
+
function stableMemoryId(type, content) {
|
|
250
|
+
return `global_${type}_${fnv1a(memoryKey(type, content))}`;
|
|
251
|
+
}
|
|
252
|
+
function fnv1a(value) {
|
|
253
|
+
let hash = 0x811c9dc5;
|
|
254
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
255
|
+
hash ^= value.charCodeAt(index);
|
|
256
|
+
hash = Math.imul(hash, 0x01000193);
|
|
257
|
+
}
|
|
258
|
+
return (hash >>> 0).toString(16).padStart(8, "0");
|
|
259
|
+
}
|
|
260
|
+
function clamp(value) {
|
|
261
|
+
return Math.max(0, Math.min(1, Number.isFinite(value) ? value : 0));
|
|
262
|
+
}
|
|
263
|
+
function unique(items) {
|
|
264
|
+
const seen = new Set();
|
|
265
|
+
return items.filter((item) => {
|
|
266
|
+
if (seen.has(item))
|
|
267
|
+
return false;
|
|
268
|
+
seen.add(item);
|
|
269
|
+
return true;
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
function isMemoryRecord(value) {
|
|
273
|
+
if (!value || typeof value !== "object")
|
|
274
|
+
return false;
|
|
275
|
+
const record = value;
|
|
276
|
+
return (typeof record.id === "string" &&
|
|
277
|
+
isMemoryType(record.type) &&
|
|
278
|
+
typeof record.content === "string" &&
|
|
279
|
+
typeof record.normalized === "string" &&
|
|
280
|
+
record.scope === "global" &&
|
|
281
|
+
isMemoryStatus(record.status) &&
|
|
282
|
+
typeof record.score?.importance === "number" &&
|
|
283
|
+
typeof record.score.confidence === "number" &&
|
|
284
|
+
isMemorySource(record.source) &&
|
|
285
|
+
Array.isArray(record.entities) &&
|
|
286
|
+
typeof record.createdAt === "string" &&
|
|
287
|
+
typeof record.updatedAt === "string");
|
|
288
|
+
}
|
|
289
|
+
function isMemoryType(value) {
|
|
290
|
+
return (value === "summary" ||
|
|
291
|
+
value === "decision" ||
|
|
292
|
+
value === "preference" ||
|
|
293
|
+
value === "open_question" ||
|
|
294
|
+
value === "artifact" ||
|
|
295
|
+
value === "timeline" ||
|
|
296
|
+
value === "fact");
|
|
297
|
+
}
|
|
298
|
+
function isMemoryStatus(value) {
|
|
299
|
+
return value === "active" || value === "stale" || value === "conflicted" || value === "superseded" || value === "archived";
|
|
300
|
+
}
|
|
301
|
+
function isMemorySource(value) {
|
|
302
|
+
if (!value || typeof value !== "object")
|
|
303
|
+
return false;
|
|
304
|
+
const source = value;
|
|
305
|
+
return source.kind === "ai_processing" || source.kind === "manual" || source.kind === "hook";
|
|
306
|
+
}
|