peon-mem 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/peon-mem.mjs +9 -2
- package/dist/compression.js +4 -3
- package/dist/config.d.ts +13 -0
- package/dist/config.js +26 -0
- package/dist/daemon.d.ts +0 -3
- package/dist/daemon.js +12 -137
- package/dist/embedding-store.d.ts +10 -0
- package/dist/embedding-store.js +97 -19
- package/dist/entity-extraction.js +4 -3
- package/dist/global-extraction.js +4 -3
- package/dist/hyde.js +4 -3
- package/dist/logger.d.ts +20 -0
- package/dist/logger.js +73 -6
- package/dist/memory-store.d.ts +5 -1
- package/dist/memory-store.js +158 -12
- package/dist/monitor.js +14 -224
- package/dist/overview.d.ts +18 -6
- package/dist/overview.js +92 -14
- package/dist/processor.d.ts +36 -0
- package/dist/processor.js +85 -2
- package/dist/quality.d.ts +17 -1
- package/dist/quality.js +152 -25
- package/dist/recuration.js +4 -3
- package/dist/reranker.js +4 -3
- package/dist/tools.d.ts +13 -0
- package/dist/tools.js +76 -14
- package/package.json +3 -3
- package/scripts/peon-operations-watch.mjs +80 -0
- package/dist/roundtable.d.ts +0 -104
- package/dist/roundtable.js +0 -787
package/dist/processor.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PeonMemoryStore } from "./memory-store.js";
|
|
2
|
-
import { loadPeonConfig } from "./config.js";
|
|
2
|
+
import { llmEnabled, loadPeonConfig } from "./config.js";
|
|
3
3
|
import { createQualityReport } from "./quality.js";
|
|
4
4
|
import { extractDomainEntitiesViaModel } from "./entity-extraction.js";
|
|
5
5
|
export class PeonMemoryProcessor {
|
|
@@ -99,7 +99,9 @@ export class PeonMemoryProcessor {
|
|
|
99
99
|
trigger: input.trigger,
|
|
100
100
|
force: input.force ?? false,
|
|
101
101
|
aiMode: this.config.aiMode,
|
|
102
|
-
|
|
102
|
+
// A local provider (Ollama) needs no key. Gating on openRouterApiKey here meant a
|
|
103
|
+
// fully-local setup skipped every automatic consolidation as "missing_api_key".
|
|
104
|
+
hasApiKey: llmEnabled(this.config),
|
|
103
105
|
hasManualAiResult: Boolean(input.aiResult)
|
|
104
106
|
});
|
|
105
107
|
if (decision.action === "skip") {
|
|
@@ -202,6 +204,11 @@ export class OpenRouterMemoryModelClient {
|
|
|
202
204
|
// Reserve explicit output room. Without this, OpenRouter applies the provider's default
|
|
203
205
|
// completion cap, which — paired with the delta cap on the input side — keeps the JSON
|
|
204
206
|
// reply from truncating mid-object. Env-tunable for very large brains.
|
|
207
|
+
// Force a JSON-object reply. A hosted model usually obeys "reply with JSON" from
|
|
208
|
+
// the prompt alone; a local 7B often answers conversationally instead ("It sounds
|
|
209
|
+
// like..."), which fails the parse and loses the whole consolidation. Ollama and
|
|
210
|
+
// the OpenAI API both honour this flag, so ask for it rather than trusting prose.
|
|
211
|
+
response_format: { type: "json_object" },
|
|
205
212
|
max_tokens: Number(process.env.PEON_CONSOLIDATION_MAX_TOKENS) || 8192
|
|
206
213
|
})
|
|
207
214
|
});
|
|
@@ -210,6 +217,19 @@ export class OpenRouterMemoryModelClient {
|
|
|
210
217
|
throw new Error(`OpenRouter memory processing failed with ${response.status}${body ? `: ${body}` : ""}`);
|
|
211
218
|
}
|
|
212
219
|
const json = (await response.json());
|
|
220
|
+
// A server whose context window is smaller than this prompt does not error: it
|
|
221
|
+
// keeps only the TAIL and silently drops the rest — which is the system prompt and
|
|
222
|
+
// the JSON schema. The model then answers without its instructions, and the result
|
|
223
|
+
// is empty or wrong. Refuse it rather than mark the session as consolidated.
|
|
224
|
+
const estimatedPromptTokens = estimatePromptTokensForTruncation(systemPrompt) + estimatePromptTokensForTruncation(userPrompt);
|
|
225
|
+
const reportedPromptTokens = json.usage?.prompt_tokens;
|
|
226
|
+
if (detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens)) {
|
|
227
|
+
throw new Error(`The model server truncated the consolidation prompt: it processed ${reportedPromptTokens} tokens ` +
|
|
228
|
+
`of roughly ${estimatedPromptTokens} sent. Its context window is too small, so the instructions and ` +
|
|
229
|
+
`schema were cut off and the result would be empty. The session log was NOT consumed and will be ` +
|
|
230
|
+
`retried. Fix: give the model a larger context window — for Ollama, create a model with ` +
|
|
231
|
+
`"PARAMETER num_ctx 32768" (or set OLLAMA_CONTEXT_LENGTH) and point PEON_PROCESSING_MODEL at it.`);
|
|
232
|
+
}
|
|
213
233
|
const content = json.choices?.[0]?.message?.content;
|
|
214
234
|
if (!content)
|
|
215
235
|
throw new Error("OpenRouter memory processing response did not include content.");
|
|
@@ -351,6 +371,69 @@ function isMemoryStatus(value) {
|
|
|
351
371
|
function estimateTokens(text) {
|
|
352
372
|
return Math.max(1, Math.ceil(text.length / 4));
|
|
353
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* Did the model server silently truncate the prompt?
|
|
376
|
+
*
|
|
377
|
+
* OpenAI-compatible servers report usage.prompt_tokens: what the model actually read.
|
|
378
|
+
* Ollama, at its 4096-token default, reports exactly 4096 for a ~17K-token prompt.
|
|
379
|
+
*
|
|
380
|
+
* The estimate comes from estimatePromptTokensForTruncation, and the threshold has to
|
|
381
|
+
* respect how rough it is. For English that estimate is chars/4, which OVER-estimates
|
|
382
|
+
* (real text runs ~5.5 chars/token), so an untruncated prompt still reports ~0.73 of
|
|
383
|
+
* it. A truncated one reports ~0.24. Below 0.5 is unambiguous: reaching it without
|
|
384
|
+
* truncation would take 8+ chars per token. Small prompts are ignored, where
|
|
385
|
+
* estimation noise is a large share of the total.
|
|
386
|
+
*/
|
|
387
|
+
export function detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens) {
|
|
388
|
+
if (!reportedPromptTokens || reportedPromptTokens <= 0)
|
|
389
|
+
return false; // no usage reported: cannot tell
|
|
390
|
+
if (estimatedPromptTokens - reportedPromptTokens < 1000)
|
|
391
|
+
return false;
|
|
392
|
+
return reportedPromptTokens / estimatedPromptTokens < 0.5;
|
|
393
|
+
}
|
|
394
|
+
/** Scripts that tokenize at roughly one token per character or more, not one per word. */
|
|
395
|
+
const TOKEN_DENSE_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Bopomofo}\u3000-\u303F\uFF00-\uFFEF]/u;
|
|
396
|
+
const TOKENS_PER_ASCII_CHAR = 1 / 4;
|
|
397
|
+
const TOKENS_PER_DENSE_CHAR = 0.75;
|
|
398
|
+
const TOKENS_PER_OTHER_CHAR = 0.35;
|
|
399
|
+
/**
|
|
400
|
+
* Prompt size estimate for detectPromptTruncation ONLY. estimateTokens (chars/4) stays
|
|
401
|
+
* the cost/reporting estimate; this one exists because chars/4 undercounts token-dense
|
|
402
|
+
* scripts, which let a truncated CJK prompt pass as untruncated.
|
|
403
|
+
*
|
|
404
|
+
* The detector flags reported/estimated < 0.5, so each weight has to sit between two
|
|
405
|
+
* limits: high enough that a truncated prompt falls below 0.5, and at most ~2x the
|
|
406
|
+
* MOST efficient tokenizer's rate, or an untruncated prompt falls below 0.5 too. That
|
|
407
|
+
* false positive is the worse failure: the session is refused on every retry.
|
|
408
|
+
*
|
|
409
|
+
* - ASCII, 1/4 per char: unchanged, so English behaves exactly as before (chars/4
|
|
410
|
+
* over-counts English ~1.4x; measured untruncated ratio 0.73, truncated 0.24).
|
|
411
|
+
* - Han, kana, hangul, bopomofo, CJK and fullwidth punctuation, 0.75 per char: efficient
|
|
412
|
+
* tokenizers run ~0.45-0.6 tokens per CJK char, giving an untruncated ratio of 0.6-0.8.
|
|
413
|
+
* Qwen2.5 on Ollama runs ~0.65, so a truncated CJK prompt now reads well under 0.5.
|
|
414
|
+
* - Any other non-ASCII, 0.35 per char: Cyrillic, Greek, Arabic, accented Latin and the
|
|
415
|
+
* like pack into ~0.22-0.35 tokens per char on large-vocabulary tokenizers. A flat 0.75
|
|
416
|
+
* here would read an untruncated Russian log at ~0.31 and block it forever.
|
|
417
|
+
*
|
|
418
|
+
* Iterates code points, so an astral character (CJK Extension B, emoji) counts once.
|
|
419
|
+
*/
|
|
420
|
+
export function estimatePromptTokensForTruncation(text) {
|
|
421
|
+
// Count per class and multiply once: summing 0.35 thousands of times drifts past the
|
|
422
|
+
// integer and Math.ceil would round it up.
|
|
423
|
+
let ascii = 0;
|
|
424
|
+
let dense = 0;
|
|
425
|
+
let other = 0;
|
|
426
|
+
for (const char of text) {
|
|
427
|
+
if ((char.codePointAt(0) ?? 0) < 0x80)
|
|
428
|
+
ascii += 1;
|
|
429
|
+
else if (TOKEN_DENSE_SCRIPT.test(char))
|
|
430
|
+
dense += 1;
|
|
431
|
+
else
|
|
432
|
+
other += 1;
|
|
433
|
+
}
|
|
434
|
+
const tokens = ascii * TOKENS_PER_ASCII_CHAR + dense * TOKENS_PER_DENSE_CHAR + other * TOKENS_PER_OTHER_CHAR;
|
|
435
|
+
return Math.max(1, Math.ceil(tokens));
|
|
436
|
+
}
|
|
354
437
|
function estimateTokensByChars(chars) {
|
|
355
438
|
return Math.max(0, Math.ceil(chars / 4));
|
|
356
439
|
}
|
package/dist/quality.d.ts
CHANGED
|
@@ -77,7 +77,23 @@ export interface SerializedMemoryQualityReport {
|
|
|
77
77
|
audit: MemoryQualityAuditSummary;
|
|
78
78
|
}
|
|
79
79
|
export declare function deduplicateMemoryRecords(records: MemoryRecord[]): DeduplicateMemoryRecordsResult;
|
|
80
|
-
export
|
|
80
|
+
export interface DetectConflictOptions {
|
|
81
|
+
/** Compare every pair (the original scan). Kept for equivalence testing. */
|
|
82
|
+
exhaustive?: boolean;
|
|
83
|
+
/** Override the ubiquitous-entity bucket cap (default MAX_ENTITY_BUCKET). */
|
|
84
|
+
maxEntityBucket?: number;
|
|
85
|
+
}
|
|
86
|
+
/** How many record pairs the last scan actually string-compared. */
|
|
87
|
+
export declare function conflictScanStats(): {
|
|
88
|
+
pairsEvaluated: number;
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Conflicts require a shared entity, so only pairs that co-occur in some entity's
|
|
92
|
+
* bucket can ever qualify. Indexing by entity skips the overwhelming majority of
|
|
93
|
+
* pairs without touching a string; everything derived per record (normalized text,
|
|
94
|
+
* token sets, entity maps) is computed once instead of once per pair.
|
|
95
|
+
*/
|
|
96
|
+
export declare function detectMemoryConflicts(records: MemoryRecord[], options?: DetectConflictOptions): MemoryConflict[];
|
|
81
97
|
export declare function markStaleMemoryRecords(records: MemoryRecord[], options?: StaleMemoryOptions): MarkStaleMemoryRecordsResult;
|
|
82
98
|
export declare function promoteMemoryRecords(records: MemoryRecord[], options?: PromoteMemoryRecordsOptions): PromoteMemoryRecordsResult;
|
|
83
99
|
export declare function applyMemoryQualityReport(records: MemoryRecord[], report: MemoryQualityReport): ApplyMemoryQualityReportResult;
|
package/dist/quality.js
CHANGED
|
@@ -25,34 +25,161 @@ export function deduplicateMemoryRecords(records) {
|
|
|
25
25
|
duplicates
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
28
|
+
/**
|
|
29
|
+
* An entity attached to more records than this is not evidence that two beliefs are
|
|
30
|
+
* about the same thing — on a research brain "peon" or "BIRD" appears on thousands of
|
|
31
|
+
* records, and pairing them all is what made this scan quadratic. Such buckets are
|
|
32
|
+
* skipped; a conflict between two beliefs whose ONLY link is a ubiquitous tag is
|
|
33
|
+
* exactly the false positive the same-topic gate already exists to suppress.
|
|
34
|
+
*/
|
|
35
|
+
const MAX_ENTITY_BUCKET = Number(process.env.PEON_CONFLICT_MAX_ENTITY_BUCKET) > 0
|
|
36
|
+
? Number(process.env.PEON_CONFLICT_MAX_ENTITY_BUCKET)
|
|
37
|
+
: 2000;
|
|
38
|
+
const OPPOSING_WORD_PAIRS = [
|
|
39
|
+
["enabled", "disabled", "opposing enabled/disabled language"],
|
|
40
|
+
["enable", "disable", "opposing enable/disable language"],
|
|
41
|
+
["allowed", "forbidden", "opposing allowed/forbidden language"],
|
|
42
|
+
["allow", "deny", "opposing allow/deny language"],
|
|
43
|
+
["required", "optional", "opposing required/optional language"],
|
|
44
|
+
["true", "false", "opposing true/false language"],
|
|
45
|
+
["yes", "no", "opposing yes/no language"],
|
|
46
|
+
["use", "avoid", "opposing use/avoid language"]
|
|
47
|
+
];
|
|
48
|
+
let lastPairsEvaluated = 0;
|
|
49
|
+
/** How many record pairs the last scan actually string-compared. */
|
|
50
|
+
export function conflictScanStats() {
|
|
51
|
+
return { pairsEvaluated: lastPairsEvaluated };
|
|
52
|
+
}
|
|
53
|
+
function prepareForConflicts(records) {
|
|
54
|
+
return records.map((record) => {
|
|
55
|
+
const entityPairs = record.entities.map((entity) => [normalizeEntity(entity), entity]);
|
|
56
|
+
const entityByNormalized = new Map();
|
|
57
|
+
// Later duplicates lose, matching the original Map-from-entries behaviour.
|
|
58
|
+
for (const [normalized, original] of entityPairs)
|
|
59
|
+
entityByNormalized.set(normalized, original);
|
|
60
|
+
const text = normalizeMemory(record.content);
|
|
61
|
+
return {
|
|
62
|
+
record,
|
|
63
|
+
entityPairs,
|
|
64
|
+
entityByNormalized,
|
|
65
|
+
normalizedEntities: new Set(entityPairs.map(([normalized]) => normalized)),
|
|
66
|
+
words: new Set(text.split(" ").filter(Boolean)),
|
|
67
|
+
topicTokens: contentTokens(record.content)
|
|
68
|
+
};
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
function sharedEntityOf(left, right) {
|
|
72
|
+
for (const [normalized, original] of left.entityPairs) {
|
|
73
|
+
const match = right.entityByNormalized.get(normalized);
|
|
74
|
+
if (match)
|
|
75
|
+
return original.trim() || match;
|
|
76
|
+
}
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Word presence via the precomputed token set. normalizeMemory() strips punctuation
|
|
81
|
+
* and collapses whitespace, so the old `(^|\s)word($|\s)` regex was exactly a test
|
|
82
|
+
* for "word is a whitespace-delimited token" — a Set lookup is equivalent, without
|
|
83
|
+
* building a RegExp per check.
|
|
84
|
+
*/
|
|
85
|
+
function opposingReasonOf(left, right) {
|
|
86
|
+
for (const [positive, negative, reason] of OPPOSING_WORD_PAIRS) {
|
|
87
|
+
if (left.words.has(positive) && right.words.has(negative))
|
|
88
|
+
return reason;
|
|
89
|
+
if (left.words.has(negative) && right.words.has(positive))
|
|
90
|
+
return reason;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
function sameTopicOf(left, right) {
|
|
95
|
+
let sharedEntities = 0;
|
|
96
|
+
for (const entity of left.normalizedEntities)
|
|
97
|
+
if (right.normalizedEntities.has(entity))
|
|
98
|
+
sharedEntities += 1;
|
|
99
|
+
if (sharedEntities >= 2)
|
|
100
|
+
return true;
|
|
101
|
+
if (left.topicTokens.size === 0 || right.topicTokens.size === 0)
|
|
102
|
+
return false;
|
|
103
|
+
let intersection = 0;
|
|
104
|
+
for (const token of left.topicTokens)
|
|
105
|
+
if (right.topicTokens.has(token))
|
|
106
|
+
intersection += 1;
|
|
107
|
+
const jaccard = intersection / (left.topicTokens.size + right.topicTokens.size - intersection);
|
|
108
|
+
return jaccard >= 0.25;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Conflicts require a shared entity, so only pairs that co-occur in some entity's
|
|
112
|
+
* bucket can ever qualify. Indexing by entity skips the overwhelming majority of
|
|
113
|
+
* pairs without touching a string; everything derived per record (normalized text,
|
|
114
|
+
* token sets, entity maps) is computed once instead of once per pair.
|
|
115
|
+
*/
|
|
116
|
+
export function detectMemoryConflicts(records, options = {}) {
|
|
117
|
+
const prepared = prepareForConflicts(records);
|
|
118
|
+
const found = [];
|
|
119
|
+
let pairsEvaluated = 0;
|
|
120
|
+
const evaluate = (leftIndex, rightIndex) => {
|
|
121
|
+
pairsEvaluated += 1;
|
|
122
|
+
const left = prepared[leftIndex];
|
|
123
|
+
const right = prepared[rightIndex];
|
|
124
|
+
const entity = sharedEntityOf(left, right);
|
|
125
|
+
if (!entity)
|
|
126
|
+
return;
|
|
127
|
+
const reason = opposingReasonOf(left, right);
|
|
128
|
+
if (!reason)
|
|
129
|
+
return;
|
|
130
|
+
// Same-topic gate. A shared entity + one opposing word-pair somewhere in two long beliefs
|
|
131
|
+
// is a weak signal: on a research brain, dozens of beliefs all mention "BIRD" and one says
|
|
132
|
+
// "use X" while an unrelated one says "avoid Y". They don't contradict — they're just both
|
|
133
|
+
// about BIRD. Require the two beliefs to actually be discussing the same thing before
|
|
134
|
+
// calling it a conflict: several shared entities, or real content overlap beyond the token.
|
|
135
|
+
if (!sameTopicOf(left, right))
|
|
136
|
+
return;
|
|
137
|
+
found.push({
|
|
138
|
+
leftIndex,
|
|
139
|
+
rightIndex,
|
|
140
|
+
conflict: { entity, leftId: left.record.id, rightId: right.record.id, reason }
|
|
141
|
+
});
|
|
142
|
+
};
|
|
143
|
+
if (options.exhaustive) {
|
|
144
|
+
for (let leftIndex = 0; leftIndex < prepared.length; leftIndex += 1) {
|
|
145
|
+
for (let rightIndex = leftIndex + 1; rightIndex < prepared.length; rightIndex += 1) {
|
|
146
|
+
evaluate(leftIndex, rightIndex);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
const byEntity = new Map();
|
|
152
|
+
for (let index = 0; index < prepared.length; index += 1) {
|
|
153
|
+
for (const entity of prepared[index].normalizedEntities) {
|
|
154
|
+
const bucket = byEntity.get(entity);
|
|
155
|
+
if (bucket)
|
|
156
|
+
bucket.push(index);
|
|
157
|
+
else
|
|
158
|
+
byEntity.set(entity, [index]);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const seen = new Set();
|
|
162
|
+
const width = prepared.length;
|
|
163
|
+
for (const bucket of byEntity.values()) {
|
|
164
|
+
if (bucket.length > (options.maxEntityBucket ?? MAX_ENTITY_BUCKET))
|
|
46
165
|
continue;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
166
|
+
for (let a = 0; a < bucket.length; a += 1) {
|
|
167
|
+
for (let b = a + 1; b < bucket.length; b += 1) {
|
|
168
|
+
const leftIndex = bucket[a];
|
|
169
|
+
const rightIndex = bucket[b];
|
|
170
|
+
const key = leftIndex * width + rightIndex;
|
|
171
|
+
if (seen.has(key))
|
|
172
|
+
continue;
|
|
173
|
+
seen.add(key);
|
|
174
|
+
evaluate(leftIndex, rightIndex);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
53
177
|
}
|
|
178
|
+
// Restore the original left-to-right emission order.
|
|
179
|
+
found.sort((x, y) => x.leftIndex - y.leftIndex || x.rightIndex - y.rightIndex);
|
|
54
180
|
}
|
|
55
|
-
|
|
181
|
+
lastPairsEvaluated = pairsEvaluated;
|
|
182
|
+
return found.map((entry) => entry.conflict);
|
|
56
183
|
}
|
|
57
184
|
export function markStaleMemoryRecords(records, options = {}) {
|
|
58
185
|
const now = options.now ?? new Date();
|
package/dist/recuration.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { llmEnabled, llmEndpoint, llmHeaders } from "./config.js";
|
|
1
2
|
const BATCH_SIZE = 30;
|
|
2
3
|
const PER_BATCH_MAX_FRACTION = 0.4; // a batch that wants to drop more than this is misjudging — skip it
|
|
3
4
|
export function createRecurator(config) {
|
|
4
|
-
if (
|
|
5
|
+
if (!llmEnabled(config))
|
|
5
6
|
return null;
|
|
6
7
|
return async (records) => {
|
|
7
8
|
const active = records.filter((r) => r.status === "active" && !r.pinned);
|
|
@@ -33,9 +34,9 @@ async function judgeBatch(config, batch) {
|
|
|
33
34
|
"KEEP every decision, result/metric, preference, file, and open question unless it is pure setup-action noise. " +
|
|
34
35
|
"You should typically remove only a small fraction; removing most beliefs is WRONG. When in any doubt, KEEP. " +
|
|
35
36
|
"Output ONLY a JSON array of the ids to remove — no fences, no prose. If none, return [].";
|
|
36
|
-
const response = await fetch(
|
|
37
|
+
const response = await fetch(llmEndpoint(config), {
|
|
37
38
|
method: "POST",
|
|
38
|
-
headers:
|
|
39
|
+
headers: llmHeaders(config),
|
|
39
40
|
body: JSON.stringify({
|
|
40
41
|
model: config.processingModel,
|
|
41
42
|
messages: [
|
package/dist/reranker.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { llmEnabled, llmEndpoint } from "./config.js";
|
|
1
2
|
const DEFAULT_TOP_K = 20;
|
|
2
3
|
const DEFAULT_SNIPPET_CHARS = 240;
|
|
3
4
|
export async function rerankRecords(query, records, options) {
|
|
@@ -6,7 +7,7 @@ export async function rerankRecords(query, records, options) {
|
|
|
6
7
|
// Cheap exits — never pay an LLM call when it cannot help.
|
|
7
8
|
if (!q || records.length < 2)
|
|
8
9
|
return records;
|
|
9
|
-
if (
|
|
10
|
+
if (!llmEnabled(config))
|
|
10
11
|
return records;
|
|
11
12
|
const topK = Math.max(2, Math.trunc(options.topK ?? DEFAULT_TOP_K));
|
|
12
13
|
const snippetChars = Math.max(40, Math.trunc(options.snippetChars ?? DEFAULT_SNIPPET_CHARS));
|
|
@@ -24,10 +25,10 @@ export async function rerankRecords(query, records, options) {
|
|
|
24
25
|
"Include every number exactly once. No prose, no code fences.";
|
|
25
26
|
const user = `Query: ${q}\n\nCandidates:\n${numbered}\n\nJSON array of numbers (most relevant first):`;
|
|
26
27
|
try {
|
|
27
|
-
const response = await doFetch(
|
|
28
|
+
const response = await doFetch(llmEndpoint(config), {
|
|
28
29
|
method: "POST",
|
|
29
30
|
headers: {
|
|
30
|
-
Authorization: `Bearer ${config.openRouterApiKey}`,
|
|
31
|
+
Authorization: `Bearer ${config.llmApiKey ?? config.openRouterApiKey ?? ""}`,
|
|
31
32
|
"Content-Type": "application/json"
|
|
32
33
|
},
|
|
33
34
|
body: JSON.stringify({
|
package/dist/tools.d.ts
CHANGED
|
@@ -229,4 +229,17 @@ export interface PeonTools {
|
|
|
229
229
|
processMemory(input: ProcessMemoryToolInput): Promise<ProcessMemoryResult>;
|
|
230
230
|
maybeProcessMemory(input: MaybeProcessMemoryToolInput): Promise<MaybeProcessMemoryResult>;
|
|
231
231
|
}
|
|
232
|
+
/** How many project stores are currently held open. */
|
|
233
|
+
export declare function openStoreCacheStats(): {
|
|
234
|
+
openStores: number;
|
|
235
|
+
};
|
|
236
|
+
/**
|
|
237
|
+
* PEON_DAEMON_URL as an http(s) base URL, or undefined to run the tools in-process.
|
|
238
|
+
*
|
|
239
|
+
* The variable is optional, and registry listings render it with a placeholder value
|
|
240
|
+
* ("your-peon-daemon-url-here"). A copied placeholder must not turn a working in-process
|
|
241
|
+
* server into one where every call fails, so anything that is not an http(s) URL is
|
|
242
|
+
* dropped with a warning on stderr (stdout carries the MCP protocol).
|
|
243
|
+
*/
|
|
244
|
+
export declare function resolveDaemonUrl(raw: string | undefined, warn?: (message: string) => void): string | undefined;
|
|
232
245
|
export declare function createPeonTools(options?: CreatePeonToolsOptions): PeonTools;
|
package/dist/tools.js
CHANGED
|
@@ -14,19 +14,69 @@ import { createRecurator } from "./recuration.js";
|
|
|
14
14
|
import { evaluatePeonProject } from "./evaluation.js";
|
|
15
15
|
import { buildContextInjection } from "./injection.js";
|
|
16
16
|
import { SessionIndex } from "./session-index.js";
|
|
17
|
+
/**
|
|
18
|
+
* Open memory stores are cached per project so repeated calls don't reopen a brain.
|
|
19
|
+
* Left unbounded, every project the daemon ever touched stayed resident — and each
|
|
20
|
+
* store carries an EmbeddingStore whose sidecar cache holds real memory. Bound it:
|
|
21
|
+
* an evicted project simply reopens on next use, which costs one open, not accuracy.
|
|
22
|
+
*/
|
|
23
|
+
const MAX_OPEN_STORES = Number(process.env.PEON_MAX_OPEN_STORES) > 0
|
|
24
|
+
? Number(process.env.PEON_MAX_OPEN_STORES)
|
|
25
|
+
: 4;
|
|
26
|
+
let openStoreCount = 0;
|
|
27
|
+
/** How many project stores are currently held open. */
|
|
28
|
+
export function openStoreCacheStats() {
|
|
29
|
+
return { openStores: openStoreCount };
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* PEON_DAEMON_URL as an http(s) base URL, or undefined to run the tools in-process.
|
|
33
|
+
*
|
|
34
|
+
* The variable is optional, and registry listings render it with a placeholder value
|
|
35
|
+
* ("your-peon-daemon-url-here"). A copied placeholder must not turn a working in-process
|
|
36
|
+
* server into one where every call fails, so anything that is not an http(s) URL is
|
|
37
|
+
* dropped with a warning on stderr (stdout carries the MCP protocol).
|
|
38
|
+
*/
|
|
39
|
+
export function resolveDaemonUrl(raw, warn = (message) => console.warn(message)) {
|
|
40
|
+
const value = raw?.trim();
|
|
41
|
+
if (!value)
|
|
42
|
+
return undefined;
|
|
43
|
+
try {
|
|
44
|
+
const { protocol } = new URL(value);
|
|
45
|
+
if (protocol === "http:" || protocol === "https:")
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
// not a URL at all: fall through to the warning
|
|
50
|
+
}
|
|
51
|
+
warn(`[peon] ignoring PEON_DAEMON_URL=${JSON.stringify(value)}: not an http(s) URL. Running the tools in-process instead.`);
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
17
54
|
export function createPeonTools(options = {}) {
|
|
18
|
-
|
|
19
|
-
|
|
55
|
+
const daemonUrl = resolveDaemonUrl(options.daemonUrl);
|
|
56
|
+
if (daemonUrl) {
|
|
57
|
+
return createDaemonBackedTools(daemonUrl);
|
|
20
58
|
}
|
|
21
59
|
const storesByProject = new Map();
|
|
22
60
|
const sessionIndex = new SessionIndex(options.sessionIndexPath);
|
|
23
61
|
let globalStorePromise;
|
|
24
62
|
async function storeFor(projectPath) {
|
|
25
63
|
const existing = storesByProject.get(projectPath);
|
|
26
|
-
if (existing)
|
|
64
|
+
if (existing) {
|
|
65
|
+
// Re-insert so Map iteration order stays least-recently-used first.
|
|
66
|
+
storesByProject.delete(projectPath);
|
|
67
|
+
storesByProject.set(projectPath, existing);
|
|
68
|
+
openStoreCount = storesByProject.size;
|
|
27
69
|
return existing;
|
|
70
|
+
}
|
|
28
71
|
const store = await PeonMemoryStore.open({ projectPath });
|
|
29
72
|
storesByProject.set(projectPath, store);
|
|
73
|
+
while (storesByProject.size > MAX_OPEN_STORES) {
|
|
74
|
+
const oldest = storesByProject.keys().next().value;
|
|
75
|
+
if (oldest === undefined)
|
|
76
|
+
break;
|
|
77
|
+
storesByProject.delete(oldest);
|
|
78
|
+
}
|
|
79
|
+
openStoreCount = storesByProject.size;
|
|
30
80
|
return store;
|
|
31
81
|
}
|
|
32
82
|
async function storeForSession(sessionId) {
|
|
@@ -366,7 +416,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
366
416
|
url.searchParams.set("query", input.query);
|
|
367
417
|
if (input.maxChars)
|
|
368
418
|
url.searchParams.set("maxChars", String(input.maxChars));
|
|
369
|
-
const response = await
|
|
419
|
+
const response = await daemonFetch(url);
|
|
370
420
|
return readJsonResponse(response);
|
|
371
421
|
},
|
|
372
422
|
async inspectBrain(input) {
|
|
@@ -376,7 +426,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
376
426
|
url.searchParams.set("query", input.query);
|
|
377
427
|
if (input.maxChars)
|
|
378
428
|
url.searchParams.set("maxChars", String(input.maxChars));
|
|
379
|
-
const response = await
|
|
429
|
+
const response = await daemonFetch(url);
|
|
380
430
|
return readJsonResponse(response);
|
|
381
431
|
},
|
|
382
432
|
async searchMemory(input) {
|
|
@@ -387,7 +437,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
387
437
|
url.searchParams.set("limit", String(input.limit));
|
|
388
438
|
if (input.maxChars)
|
|
389
439
|
url.searchParams.set("maxChars", String(input.maxChars));
|
|
390
|
-
const response = await
|
|
440
|
+
const response = await daemonFetch(url);
|
|
391
441
|
return readJsonResponse(response);
|
|
392
442
|
},
|
|
393
443
|
async qualityReport(input) {
|
|
@@ -395,7 +445,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
395
445
|
url.searchParams.set("projectPath", input.projectPath);
|
|
396
446
|
if (input.staleAfterDays)
|
|
397
447
|
url.searchParams.set("staleAfterDays", String(input.staleAfterDays));
|
|
398
|
-
const response = await
|
|
448
|
+
const response = await daemonFetch(url);
|
|
399
449
|
return readJsonResponse(response);
|
|
400
450
|
},
|
|
401
451
|
async rememberGlobal(input) {
|
|
@@ -409,7 +459,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
409
459
|
url.searchParams.set("type", input.type);
|
|
410
460
|
if (input.status)
|
|
411
461
|
url.searchParams.set("status", input.status);
|
|
412
|
-
const response = await
|
|
462
|
+
const response = await daemonFetch(url);
|
|
413
463
|
return readJsonResponse(response);
|
|
414
464
|
},
|
|
415
465
|
async importGlobalMemory(input) {
|
|
@@ -434,17 +484,17 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
434
484
|
const url = new URL(`${baseUrl}/brain/activity`);
|
|
435
485
|
if (input.limit)
|
|
436
486
|
url.searchParams.set("limit", String(input.limit));
|
|
437
|
-
return readJsonResponse(await
|
|
487
|
+
return readJsonResponse(await daemonFetch(url));
|
|
438
488
|
},
|
|
439
489
|
async globalDashboard() {
|
|
440
|
-
return readJsonResponse(await
|
|
490
|
+
return readJsonResponse(await daemonFetch(`${baseUrl}/global/dashboard`));
|
|
441
491
|
},
|
|
442
492
|
async brainActions(input) {
|
|
443
493
|
const url = new URL(`${baseUrl}/brain/actions`);
|
|
444
494
|
url.searchParams.set("projectPath", input.projectPath);
|
|
445
495
|
if (input.limit)
|
|
446
496
|
url.searchParams.set("limit", String(input.limit));
|
|
447
|
-
return readJsonResponse(await
|
|
497
|
+
return readJsonResponse(await daemonFetch(url));
|
|
448
498
|
},
|
|
449
499
|
async restoreBackup(input) {
|
|
450
500
|
return postJson(`${baseUrl}/brain/restore`, input);
|
|
@@ -470,7 +520,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
470
520
|
url.searchParams.set("maxChars", String(input.maxChars));
|
|
471
521
|
if (input.includeInactive)
|
|
472
522
|
url.searchParams.set("includeInactive", "true");
|
|
473
|
-
const response = await
|
|
523
|
+
const response = await daemonFetch(url);
|
|
474
524
|
return readJsonResponse(response);
|
|
475
525
|
},
|
|
476
526
|
async crossProjectSearch(input) {
|
|
@@ -486,7 +536,7 @@ function createDaemonBackedTools(daemonUrl) {
|
|
|
486
536
|
url.searchParams.set("limit", String(input.limit));
|
|
487
537
|
if (input.maxProjects)
|
|
488
538
|
url.searchParams.set("maxProjects", String(input.maxProjects));
|
|
489
|
-
const response = await
|
|
539
|
+
const response = await daemonFetch(url);
|
|
490
540
|
return readJsonResponse(response);
|
|
491
541
|
},
|
|
492
542
|
async processMemory(input) {
|
|
@@ -528,13 +578,25 @@ function formatSearchInjectionPreview(records) {
|
|
|
528
578
|
return ["Peon Search Results", ...records.map(formatRankedMemoryRecord)].join("\n");
|
|
529
579
|
}
|
|
530
580
|
async function postJson(url, body) {
|
|
531
|
-
const response = await
|
|
581
|
+
const response = await daemonFetch(url, {
|
|
532
582
|
method: "POST",
|
|
533
583
|
headers: { "content-type": "application/json" },
|
|
534
584
|
body: JSON.stringify(body)
|
|
535
585
|
});
|
|
536
586
|
return readJsonResponse(response);
|
|
537
587
|
}
|
|
588
|
+
/** fetch against the daemon, with an actionable error when nothing answers there. */
|
|
589
|
+
async function daemonFetch(url, init) {
|
|
590
|
+
try {
|
|
591
|
+
return await fetch(url, init);
|
|
592
|
+
}
|
|
593
|
+
catch (error) {
|
|
594
|
+
const reason = error.cause?.code ?? (error instanceof Error ? error.message : String(error));
|
|
595
|
+
throw new Error(`Peon daemon is not reachable at ${new URL(String(url)).origin} (${reason}). Start it with ` +
|
|
596
|
+
`"npx peon-mem install" (sets it up as a background service) or "npx peon-mem daemon", ` +
|
|
597
|
+
`or unset PEON_DAEMON_URL to run the tools in-process.`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
538
600
|
async function readJsonResponse(response) {
|
|
539
601
|
const text = await response.text();
|
|
540
602
|
const body = text ? JSON.parse(text) : {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "peon-mem",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.9",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"prepublishOnly": "npm run build && npm test"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
22
|
+
"@modelcontextprotocol/sdk": "^1.29.0"
|
|
23
23
|
},
|
|
24
24
|
"description": "Local-first hierarchical memory brain for AI coding agents — auto-capture, LLM consolidation into beliefs, hybrid retrieval, per-prompt injection, a living Neural Universe monitor, and a daily self-improvement loop. Works with Claude Code, Codex, and any MCP client.",
|
|
25
25
|
"license": "MIT",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"devDependencies": {
|
|
45
45
|
"typescript": "^5.7.0",
|
|
46
|
-
"vitest": "^4.
|
|
46
|
+
"vitest": "^4.1.11",
|
|
47
47
|
"@types/node": "^22.0.0"
|
|
48
48
|
},
|
|
49
49
|
"files": [
|