peon-mem 1.0.7 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compression.js +4 -3
- package/dist/config.d.ts +13 -0
- package/dist/config.js +26 -0
- package/dist/daemon.d.ts +0 -3
- package/dist/daemon.js +12 -137
- package/dist/embedding-store.d.ts +10 -0
- package/dist/embedding-store.js +97 -19
- package/dist/entity-extraction.js +4 -3
- package/dist/global-extraction.js +4 -3
- package/dist/hyde.js +4 -3
- package/dist/logger.d.ts +20 -0
- package/dist/logger.js +73 -6
- package/dist/memory-store.d.ts +5 -1
- package/dist/memory-store.js +158 -12
- package/dist/monitor.js +14 -224
- package/dist/overview.d.ts +18 -6
- package/dist/overview.js +92 -14
- package/dist/processor.d.ts +36 -0
- package/dist/processor.js +85 -2
- package/dist/quality.d.ts +17 -1
- package/dist/quality.js +152 -25
- package/dist/recuration.js +4 -3
- package/dist/reranker.js +4 -3
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +27 -1
- package/package.json +3 -3
- package/scripts/peon-operations-watch.mjs +80 -0
- package/dist/roundtable.d.ts +0 -104
- package/dist/roundtable.js +0 -787
package/dist/processor.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PeonMemoryStore } from "./memory-store.js";
|
|
2
|
-
import { loadPeonConfig } from "./config.js";
|
|
2
|
+
import { llmEnabled, loadPeonConfig } from "./config.js";
|
|
3
3
|
import { createQualityReport } from "./quality.js";
|
|
4
4
|
import { extractDomainEntitiesViaModel } from "./entity-extraction.js";
|
|
5
5
|
export class PeonMemoryProcessor {
|
|
@@ -99,7 +99,9 @@ export class PeonMemoryProcessor {
|
|
|
99
99
|
trigger: input.trigger,
|
|
100
100
|
force: input.force ?? false,
|
|
101
101
|
aiMode: this.config.aiMode,
|
|
102
|
-
|
|
102
|
+
// A local provider (Ollama) needs no key. Gating on openRouterApiKey here meant a
|
|
103
|
+
// fully-local setup skipped every automatic consolidation as "missing_api_key".
|
|
104
|
+
hasApiKey: llmEnabled(this.config),
|
|
103
105
|
hasManualAiResult: Boolean(input.aiResult)
|
|
104
106
|
});
|
|
105
107
|
if (decision.action === "skip") {
|
|
@@ -202,6 +204,11 @@ export class OpenRouterMemoryModelClient {
|
|
|
202
204
|
// Reserve explicit output room. Without this, OpenRouter applies the provider's default
|
|
203
205
|
// completion cap, which — paired with the delta cap on the input side — keeps the JSON
|
|
204
206
|
// reply from truncating mid-object. Env-tunable for very large brains.
|
|
207
|
+
// Force a JSON-object reply. A hosted model usually obeys "reply with JSON" from
|
|
208
|
+
// the prompt alone; a local 7B often answers conversationally instead ("It sounds
|
|
209
|
+
// like..."), which fails the parse and loses the whole consolidation. Ollama and
|
|
210
|
+
// the OpenAI API both honour this flag, so ask for it rather than trusting prose.
|
|
211
|
+
response_format: { type: "json_object" },
|
|
205
212
|
max_tokens: Number(process.env.PEON_CONSOLIDATION_MAX_TOKENS) || 8192
|
|
206
213
|
})
|
|
207
214
|
});
|
|
@@ -210,6 +217,19 @@ export class OpenRouterMemoryModelClient {
|
|
|
210
217
|
throw new Error(`OpenRouter memory processing failed with ${response.status}${body ? `: ${body}` : ""}`);
|
|
211
218
|
}
|
|
212
219
|
const json = (await response.json());
|
|
220
|
+
// A server whose context window is smaller than this prompt does not error: it
|
|
221
|
+
// keeps only the TAIL and silently drops the rest — which is the system prompt and
|
|
222
|
+
// the JSON schema. The model then answers without its instructions, and the result
|
|
223
|
+
// is empty or wrong. Refuse it rather than mark the session as consolidated.
|
|
224
|
+
const estimatedPromptTokens = estimatePromptTokensForTruncation(systemPrompt) + estimatePromptTokensForTruncation(userPrompt);
|
|
225
|
+
const reportedPromptTokens = json.usage?.prompt_tokens;
|
|
226
|
+
if (detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens)) {
|
|
227
|
+
throw new Error(`The model server truncated the consolidation prompt: it processed ${reportedPromptTokens} tokens ` +
|
|
228
|
+
`of roughly ${estimatedPromptTokens} sent. Its context window is too small, so the instructions and ` +
|
|
229
|
+
`schema were cut off and the result would be empty. The session log was NOT consumed and will be ` +
|
|
230
|
+
`retried. Fix: give the model a larger context window — for Ollama, create a model with ` +
|
|
231
|
+
`"PARAMETER num_ctx 32768" (or set OLLAMA_CONTEXT_LENGTH) and point PEON_PROCESSING_MODEL at it.`);
|
|
232
|
+
}
|
|
213
233
|
const content = json.choices?.[0]?.message?.content;
|
|
214
234
|
if (!content)
|
|
215
235
|
throw new Error("OpenRouter memory processing response did not include content.");
|
|
@@ -351,6 +371,69 @@ function isMemoryStatus(value) {
|
|
|
351
371
|
function estimateTokens(text) {
|
|
352
372
|
return Math.max(1, Math.ceil(text.length / 4));
|
|
353
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* Did the model server silently truncate the prompt?
|
|
376
|
+
*
|
|
377
|
+
* OpenAI-compatible servers report usage.prompt_tokens: what the model actually read.
|
|
378
|
+
* Ollama, at its 4096-token default, reports exactly 4096 for a ~17K-token prompt.
|
|
379
|
+
*
|
|
380
|
+
* The estimate comes from estimatePromptTokensForTruncation, and the threshold has to
|
|
381
|
+
* respect how rough it is. For English that estimate is chars/4, which OVER-estimates
|
|
382
|
+
* (real text runs ~5.5 chars/token), so an untruncated prompt still reports ~0.73 of
|
|
383
|
+
* it. A truncated one reports ~0.24. Below 0.5 is unambiguous: reaching it without
|
|
384
|
+
* truncation would take 8+ chars per token. Small prompts are ignored, where
|
|
385
|
+
* estimation noise is a large share of the total.
|
|
386
|
+
*/
|
|
387
|
+
export function detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens) {
|
|
388
|
+
if (!reportedPromptTokens || reportedPromptTokens <= 0)
|
|
389
|
+
return false; // no usage reported: cannot tell
|
|
390
|
+
if (estimatedPromptTokens - reportedPromptTokens < 1000)
|
|
391
|
+
return false;
|
|
392
|
+
return reportedPromptTokens / estimatedPromptTokens < 0.5;
|
|
393
|
+
}
|
|
394
|
+
/** Scripts that tokenize at roughly one token per character or more, not one per word. */
|
|
395
|
+
const TOKEN_DENSE_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Bopomofo}\u3000-\u303F\uFF00-\uFFEF]/u;
|
|
396
|
+
const TOKENS_PER_ASCII_CHAR = 1 / 4;
|
|
397
|
+
const TOKENS_PER_DENSE_CHAR = 0.75;
|
|
398
|
+
const TOKENS_PER_OTHER_CHAR = 0.35;
|
|
399
|
+
/**
|
|
400
|
+
* Prompt size estimate for detectPromptTruncation ONLY. estimateTokens (chars/4) stays
|
|
401
|
+
* the cost/reporting estimate; this one exists because chars/4 undercounts token-dense
|
|
402
|
+
* scripts, which let a truncated CJK prompt pass as untruncated.
|
|
403
|
+
*
|
|
404
|
+
* The detector flags reported/estimated < 0.5, so each weight has to sit between two
|
|
405
|
+
* limits: high enough that a truncated prompt falls below 0.5, and at most ~2x the
|
|
406
|
+
* MOST efficient tokenizer's rate, or an untruncated prompt falls below 0.5 too. That
|
|
407
|
+
* false positive is the worse failure: the session is refused on every retry.
|
|
408
|
+
*
|
|
409
|
+
* - ASCII, 1/4 per char: unchanged, so English behaves exactly as before (chars/4
|
|
410
|
+
* over-counts English ~1.4x; measured untruncated ratio 0.73, truncated 0.24).
|
|
411
|
+
* - Han, kana, hangul, bopomofo, CJK and fullwidth punctuation, 0.75 per char: efficient
|
|
412
|
+
* tokenizers run ~0.45-0.6 tokens per CJK char, giving an untruncated ratio of 0.6-0.8.
|
|
413
|
+
* Qwen2.5 on Ollama runs ~0.65, so a truncated CJK prompt now reads well under 0.5.
|
|
414
|
+
* - Any other non-ASCII, 0.35 per char: Cyrillic, Greek, Arabic, accented Latin and the
|
|
415
|
+
* like pack into ~0.22-0.35 tokens per char on large-vocabulary tokenizers. A flat 0.75
|
|
416
|
+
* here would read an untruncated Russian log at ~0.31 and block it forever.
|
|
417
|
+
*
|
|
418
|
+
* Iterates code points, so an astral character (CJK Extension B, emoji) counts once.
|
|
419
|
+
*/
|
|
420
|
+
export function estimatePromptTokensForTruncation(text) {
|
|
421
|
+
// Count per class and multiply once: summing 0.35 thousands of times drifts past the
|
|
422
|
+
// integer and Math.ceil would round it up.
|
|
423
|
+
let ascii = 0;
|
|
424
|
+
let dense = 0;
|
|
425
|
+
let other = 0;
|
|
426
|
+
for (const char of text) {
|
|
427
|
+
if ((char.codePointAt(0) ?? 0) < 0x80)
|
|
428
|
+
ascii += 1;
|
|
429
|
+
else if (TOKEN_DENSE_SCRIPT.test(char))
|
|
430
|
+
dense += 1;
|
|
431
|
+
else
|
|
432
|
+
other += 1;
|
|
433
|
+
}
|
|
434
|
+
const tokens = ascii * TOKENS_PER_ASCII_CHAR + dense * TOKENS_PER_DENSE_CHAR + other * TOKENS_PER_OTHER_CHAR;
|
|
435
|
+
return Math.max(1, Math.ceil(tokens));
|
|
436
|
+
}
|
|
354
437
|
function estimateTokensByChars(chars) {
|
|
355
438
|
return Math.max(0, Math.ceil(chars / 4));
|
|
356
439
|
}
|
package/dist/quality.d.ts
CHANGED
|
@@ -77,7 +77,23 @@ export interface SerializedMemoryQualityReport {
|
|
|
77
77
|
audit: MemoryQualityAuditSummary;
|
|
78
78
|
}
|
|
79
79
|
export declare function deduplicateMemoryRecords(records: MemoryRecord[]): DeduplicateMemoryRecordsResult;
|
|
80
|
-
export
|
|
80
|
+
export interface DetectConflictOptions {
|
|
81
|
+
/** Compare every pair (the original scan). Kept for equivalence testing. */
|
|
82
|
+
exhaustive?: boolean;
|
|
83
|
+
/** Override the ubiquitous-entity bucket cap (default MAX_ENTITY_BUCKET). */
|
|
84
|
+
maxEntityBucket?: number;
|
|
85
|
+
}
|
|
86
|
+
/** How many record pairs the last scan actually string-compared. */
|
|
87
|
+
export declare function conflictScanStats(): {
|
|
88
|
+
pairsEvaluated: number;
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Conflicts require a shared entity, so only pairs that co-occur in some entity's
|
|
92
|
+
* bucket can ever qualify. Indexing by entity skips the overwhelming majority of
|
|
93
|
+
* pairs without touching a string; everything derived per record (normalized text,
|
|
94
|
+
* token sets, entity maps) is computed once instead of once per pair.
|
|
95
|
+
*/
|
|
96
|
+
export declare function detectMemoryConflicts(records: MemoryRecord[], options?: DetectConflictOptions): MemoryConflict[];
|
|
81
97
|
export declare function markStaleMemoryRecords(records: MemoryRecord[], options?: StaleMemoryOptions): MarkStaleMemoryRecordsResult;
|
|
82
98
|
export declare function promoteMemoryRecords(records: MemoryRecord[], options?: PromoteMemoryRecordsOptions): PromoteMemoryRecordsResult;
|
|
83
99
|
export declare function applyMemoryQualityReport(records: MemoryRecord[], report: MemoryQualityReport): ApplyMemoryQualityReportResult;
|
package/dist/quality.js
CHANGED
|
@@ -25,34 +25,161 @@ export function deduplicateMemoryRecords(records) {
|
|
|
25
25
|
duplicates
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
28
|
+
/**
|
|
29
|
+
* An entity attached to more records than this is not evidence that two beliefs are
|
|
30
|
+
* about the same thing — on a research brain "peon" or "BIRD" appears on thousands of
|
|
31
|
+
* records, and pairing them all is what made this scan quadratic. Such buckets are
|
|
32
|
+
* skipped; a conflict between two beliefs whose ONLY link is a ubiquitous tag is
|
|
33
|
+
* exactly the false positive the same-topic gate already exists to suppress.
|
|
34
|
+
*/
|
|
35
|
+
const MAX_ENTITY_BUCKET = Number(process.env.PEON_CONFLICT_MAX_ENTITY_BUCKET) > 0
|
|
36
|
+
? Number(process.env.PEON_CONFLICT_MAX_ENTITY_BUCKET)
|
|
37
|
+
: 2000;
|
|
38
|
+
const OPPOSING_WORD_PAIRS = [
|
|
39
|
+
["enabled", "disabled", "opposing enabled/disabled language"],
|
|
40
|
+
["enable", "disable", "opposing enable/disable language"],
|
|
41
|
+
["allowed", "forbidden", "opposing allowed/forbidden language"],
|
|
42
|
+
["allow", "deny", "opposing allow/deny language"],
|
|
43
|
+
["required", "optional", "opposing required/optional language"],
|
|
44
|
+
["true", "false", "opposing true/false language"],
|
|
45
|
+
["yes", "no", "opposing yes/no language"],
|
|
46
|
+
["use", "avoid", "opposing use/avoid language"]
|
|
47
|
+
];
|
|
48
|
+
let lastPairsEvaluated = 0;
|
|
49
|
+
/** How many record pairs the last scan actually string-compared. */
|
|
50
|
+
export function conflictScanStats() {
|
|
51
|
+
return { pairsEvaluated: lastPairsEvaluated };
|
|
52
|
+
}
|
|
53
|
+
function prepareForConflicts(records) {
|
|
54
|
+
return records.map((record) => {
|
|
55
|
+
const entityPairs = record.entities.map((entity) => [normalizeEntity(entity), entity]);
|
|
56
|
+
const entityByNormalized = new Map();
|
|
57
|
+
// Later duplicates lose, matching the original Map-from-entries behaviour.
|
|
58
|
+
for (const [normalized, original] of entityPairs)
|
|
59
|
+
entityByNormalized.set(normalized, original);
|
|
60
|
+
const text = normalizeMemory(record.content);
|
|
61
|
+
return {
|
|
62
|
+
record,
|
|
63
|
+
entityPairs,
|
|
64
|
+
entityByNormalized,
|
|
65
|
+
normalizedEntities: new Set(entityPairs.map(([normalized]) => normalized)),
|
|
66
|
+
words: new Set(text.split(" ").filter(Boolean)),
|
|
67
|
+
topicTokens: contentTokens(record.content)
|
|
68
|
+
};
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
function sharedEntityOf(left, right) {
|
|
72
|
+
for (const [normalized, original] of left.entityPairs) {
|
|
73
|
+
const match = right.entityByNormalized.get(normalized);
|
|
74
|
+
if (match)
|
|
75
|
+
return original.trim() || match;
|
|
76
|
+
}
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Word presence via the precomputed token set. normalizeMemory() strips punctuation
|
|
81
|
+
* and collapses whitespace, so the old `(^|\s)word($|\s)` regex was exactly a test
|
|
82
|
+
* for "word is a whitespace-delimited token" — a Set lookup is equivalent, without
|
|
83
|
+
* building a RegExp per check.
|
|
84
|
+
*/
|
|
85
|
+
function opposingReasonOf(left, right) {
|
|
86
|
+
for (const [positive, negative, reason] of OPPOSING_WORD_PAIRS) {
|
|
87
|
+
if (left.words.has(positive) && right.words.has(negative))
|
|
88
|
+
return reason;
|
|
89
|
+
if (left.words.has(negative) && right.words.has(positive))
|
|
90
|
+
return reason;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
function sameTopicOf(left, right) {
|
|
95
|
+
let sharedEntities = 0;
|
|
96
|
+
for (const entity of left.normalizedEntities)
|
|
97
|
+
if (right.normalizedEntities.has(entity))
|
|
98
|
+
sharedEntities += 1;
|
|
99
|
+
if (sharedEntities >= 2)
|
|
100
|
+
return true;
|
|
101
|
+
if (left.topicTokens.size === 0 || right.topicTokens.size === 0)
|
|
102
|
+
return false;
|
|
103
|
+
let intersection = 0;
|
|
104
|
+
for (const token of left.topicTokens)
|
|
105
|
+
if (right.topicTokens.has(token))
|
|
106
|
+
intersection += 1;
|
|
107
|
+
const jaccard = intersection / (left.topicTokens.size + right.topicTokens.size - intersection);
|
|
108
|
+
return jaccard >= 0.25;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Conflicts require a shared entity, so only pairs that co-occur in some entity's
|
|
112
|
+
* bucket can ever qualify. Indexing by entity skips the overwhelming majority of
|
|
113
|
+
* pairs without touching a string; everything derived per record (normalized text,
|
|
114
|
+
* token sets, entity maps) is computed once instead of once per pair.
|
|
115
|
+
*/
|
|
116
|
+
export function detectMemoryConflicts(records, options = {}) {
|
|
117
|
+
const prepared = prepareForConflicts(records);
|
|
118
|
+
const found = [];
|
|
119
|
+
let pairsEvaluated = 0;
|
|
120
|
+
const evaluate = (leftIndex, rightIndex) => {
|
|
121
|
+
pairsEvaluated += 1;
|
|
122
|
+
const left = prepared[leftIndex];
|
|
123
|
+
const right = prepared[rightIndex];
|
|
124
|
+
const entity = sharedEntityOf(left, right);
|
|
125
|
+
if (!entity)
|
|
126
|
+
return;
|
|
127
|
+
const reason = opposingReasonOf(left, right);
|
|
128
|
+
if (!reason)
|
|
129
|
+
return;
|
|
130
|
+
// Same-topic gate. A shared entity + one opposing word-pair somewhere in two long beliefs
|
|
131
|
+
// is a weak signal: on a research brain, dozens of beliefs all mention "BIRD" and one says
|
|
132
|
+
// "use X" while an unrelated one says "avoid Y". They don't contradict — they're just both
|
|
133
|
+
// about BIRD. Require the two beliefs to actually be discussing the same thing before
|
|
134
|
+
// calling it a conflict: several shared entities, or real content overlap beyond the token.
|
|
135
|
+
if (!sameTopicOf(left, right))
|
|
136
|
+
return;
|
|
137
|
+
found.push({
|
|
138
|
+
leftIndex,
|
|
139
|
+
rightIndex,
|
|
140
|
+
conflict: { entity, leftId: left.record.id, rightId: right.record.id, reason }
|
|
141
|
+
});
|
|
142
|
+
};
|
|
143
|
+
if (options.exhaustive) {
|
|
144
|
+
for (let leftIndex = 0; leftIndex < prepared.length; leftIndex += 1) {
|
|
145
|
+
for (let rightIndex = leftIndex + 1; rightIndex < prepared.length; rightIndex += 1) {
|
|
146
|
+
evaluate(leftIndex, rightIndex);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
const byEntity = new Map();
|
|
152
|
+
for (let index = 0; index < prepared.length; index += 1) {
|
|
153
|
+
for (const entity of prepared[index].normalizedEntities) {
|
|
154
|
+
const bucket = byEntity.get(entity);
|
|
155
|
+
if (bucket)
|
|
156
|
+
bucket.push(index);
|
|
157
|
+
else
|
|
158
|
+
byEntity.set(entity, [index]);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const seen = new Set();
|
|
162
|
+
const width = prepared.length;
|
|
163
|
+
for (const bucket of byEntity.values()) {
|
|
164
|
+
if (bucket.length > (options.maxEntityBucket ?? MAX_ENTITY_BUCKET))
|
|
46
165
|
continue;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
166
|
+
for (let a = 0; a < bucket.length; a += 1) {
|
|
167
|
+
for (let b = a + 1; b < bucket.length; b += 1) {
|
|
168
|
+
const leftIndex = bucket[a];
|
|
169
|
+
const rightIndex = bucket[b];
|
|
170
|
+
const key = leftIndex * width + rightIndex;
|
|
171
|
+
if (seen.has(key))
|
|
172
|
+
continue;
|
|
173
|
+
seen.add(key);
|
|
174
|
+
evaluate(leftIndex, rightIndex);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
53
177
|
}
|
|
178
|
+
// Restore the original left-to-right emission order.
|
|
179
|
+
found.sort((x, y) => x.leftIndex - y.leftIndex || x.rightIndex - y.rightIndex);
|
|
54
180
|
}
|
|
55
|
-
|
|
181
|
+
lastPairsEvaluated = pairsEvaluated;
|
|
182
|
+
return found.map((entry) => entry.conflict);
|
|
56
183
|
}
|
|
57
184
|
export function markStaleMemoryRecords(records, options = {}) {
|
|
58
185
|
const now = options.now ?? new Date();
|
package/dist/recuration.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { llmEnabled, llmEndpoint, llmHeaders } from "./config.js";
|
|
1
2
|
const BATCH_SIZE = 30;
|
|
2
3
|
const PER_BATCH_MAX_FRACTION = 0.4; // a batch that wants to drop more than this is misjudging — skip it
|
|
3
4
|
export function createRecurator(config) {
|
|
4
|
-
if (
|
|
5
|
+
if (!llmEnabled(config))
|
|
5
6
|
return null;
|
|
6
7
|
return async (records) => {
|
|
7
8
|
const active = records.filter((r) => r.status === "active" && !r.pinned);
|
|
@@ -33,9 +34,9 @@ async function judgeBatch(config, batch) {
|
|
|
33
34
|
"KEEP every decision, result/metric, preference, file, and open question unless it is pure setup-action noise. " +
|
|
34
35
|
"You should typically remove only a small fraction; removing most beliefs is WRONG. When in any doubt, KEEP. " +
|
|
35
36
|
"Output ONLY a JSON array of the ids to remove — no fences, no prose. If none, return [].";
|
|
36
|
-
const response = await fetch(
|
|
37
|
+
const response = await fetch(llmEndpoint(config), {
|
|
37
38
|
method: "POST",
|
|
38
|
-
headers:
|
|
39
|
+
headers: llmHeaders(config),
|
|
39
40
|
body: JSON.stringify({
|
|
40
41
|
model: config.processingModel,
|
|
41
42
|
messages: [
|
package/dist/reranker.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { llmEnabled, llmEndpoint } from "./config.js";
|
|
1
2
|
const DEFAULT_TOP_K = 20;
|
|
2
3
|
const DEFAULT_SNIPPET_CHARS = 240;
|
|
3
4
|
export async function rerankRecords(query, records, options) {
|
|
@@ -6,7 +7,7 @@ export async function rerankRecords(query, records, options) {
|
|
|
6
7
|
// Cheap exits — never pay an LLM call when it cannot help.
|
|
7
8
|
if (!q || records.length < 2)
|
|
8
9
|
return records;
|
|
9
|
-
if (
|
|
10
|
+
if (!llmEnabled(config))
|
|
10
11
|
return records;
|
|
11
12
|
const topK = Math.max(2, Math.trunc(options.topK ?? DEFAULT_TOP_K));
|
|
12
13
|
const snippetChars = Math.max(40, Math.trunc(options.snippetChars ?? DEFAULT_SNIPPET_CHARS));
|
|
@@ -24,10 +25,10 @@ export async function rerankRecords(query, records, options) {
|
|
|
24
25
|
"Include every number exactly once. No prose, no code fences.";
|
|
25
26
|
const user = `Query: ${q}\n\nCandidates:\n${numbered}\n\nJSON array of numbers (most relevant first):`;
|
|
26
27
|
try {
|
|
27
|
-
const response = await doFetch(
|
|
28
|
+
const response = await doFetch(llmEndpoint(config), {
|
|
28
29
|
method: "POST",
|
|
29
30
|
headers: {
|
|
30
|
-
Authorization: `Bearer ${config.openRouterApiKey}`,
|
|
31
|
+
Authorization: `Bearer ${config.llmApiKey ?? config.openRouterApiKey ?? ""}`,
|
|
31
32
|
"Content-Type": "application/json"
|
|
32
33
|
},
|
|
33
34
|
body: JSON.stringify({
|
package/dist/tools.d.ts
CHANGED
|
@@ -229,4 +229,8 @@ export interface PeonTools {
|
|
|
229
229
|
processMemory(input: ProcessMemoryToolInput): Promise<ProcessMemoryResult>;
|
|
230
230
|
maybeProcessMemory(input: MaybeProcessMemoryToolInput): Promise<MaybeProcessMemoryResult>;
|
|
231
231
|
}
|
|
232
|
+
/** How many project stores are currently held open. */
|
|
233
|
+
export declare function openStoreCacheStats(): {
|
|
234
|
+
openStores: number;
|
|
235
|
+
};
|
|
232
236
|
export declare function createPeonTools(options?: CreatePeonToolsOptions): PeonTools;
|
package/dist/tools.js
CHANGED
|
@@ -14,6 +14,20 @@ import { createRecurator } from "./recuration.js";
|
|
|
14
14
|
import { evaluatePeonProject } from "./evaluation.js";
|
|
15
15
|
import { buildContextInjection } from "./injection.js";
|
|
16
16
|
import { SessionIndex } from "./session-index.js";
|
|
17
|
+
/**
|
|
18
|
+
* Open memory stores are cached per project so repeated calls don't reopen a brain.
|
|
19
|
+
* Left unbounded, every project the daemon ever touched stayed resident — and each
|
|
20
|
+
* store carries an EmbeddingStore whose sidecar cache holds real memory. Bound it:
|
|
21
|
+
* an evicted project simply reopens on next use, which costs one open, not accuracy.
|
|
22
|
+
*/
|
|
23
|
+
const MAX_OPEN_STORES = Number(process.env.PEON_MAX_OPEN_STORES) > 0
|
|
24
|
+
? Number(process.env.PEON_MAX_OPEN_STORES)
|
|
25
|
+
: 4;
|
|
26
|
+
let openStoreCount = 0;
|
|
27
|
+
/** How many project stores are currently held open. */
|
|
28
|
+
export function openStoreCacheStats() {
|
|
29
|
+
return { openStores: openStoreCount };
|
|
30
|
+
}
|
|
17
31
|
export function createPeonTools(options = {}) {
|
|
18
32
|
if (options.daemonUrl) {
|
|
19
33
|
return createDaemonBackedTools(options.daemonUrl);
|
|
@@ -23,10 +37,22 @@ export function createPeonTools(options = {}) {
|
|
|
23
37
|
let globalStorePromise;
|
|
24
38
|
async function storeFor(projectPath) {
|
|
25
39
|
const existing = storesByProject.get(projectPath);
|
|
26
|
-
if (existing)
|
|
40
|
+
if (existing) {
|
|
41
|
+
// Re-insert so Map iteration order stays least-recently-used first.
|
|
42
|
+
storesByProject.delete(projectPath);
|
|
43
|
+
storesByProject.set(projectPath, existing);
|
|
44
|
+
openStoreCount = storesByProject.size;
|
|
27
45
|
return existing;
|
|
46
|
+
}
|
|
28
47
|
const store = await PeonMemoryStore.open({ projectPath });
|
|
29
48
|
storesByProject.set(projectPath, store);
|
|
49
|
+
while (storesByProject.size > MAX_OPEN_STORES) {
|
|
50
|
+
const oldest = storesByProject.keys().next().value;
|
|
51
|
+
if (oldest === undefined)
|
|
52
|
+
break;
|
|
53
|
+
storesByProject.delete(oldest);
|
|
54
|
+
}
|
|
55
|
+
openStoreCount = storesByProject.size;
|
|
30
56
|
return store;
|
|
31
57
|
}
|
|
32
58
|
async function storeForSession(sessionId) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "peon-mem",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"prepublishOnly": "npm run build && npm test"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
22
|
+
"@modelcontextprotocol/sdk": "^1.29.0"
|
|
23
23
|
},
|
|
24
24
|
"description": "Local-first hierarchical memory brain for AI coding agents — auto-capture, LLM consolidation into beliefs, hybrid retrieval, per-prompt injection, a living Neural Universe monitor, and a daily self-improvement loop. Works with Claude Code, Codex, and any MCP client.",
|
|
25
25
|
"license": "MIT",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"devDependencies": {
|
|
45
45
|
"typescript": "^5.7.0",
|
|
46
|
-
"vitest": "^4.
|
|
46
|
+
"vitest": "^4.1.11",
|
|
47
47
|
"@types/node": "^22.0.0"
|
|
48
48
|
},
|
|
49
49
|
"files": [
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Is the consolidation model actually REVISING beliefs, or only appending them?
|
|
4
|
+
*
|
|
5
|
+
* Peon's value rests on supersede/obsolete operations — the difference between a
|
|
6
|
+
* brain and a filing cabinet. A model that emits none still looks healthy: runs
|
|
7
|
+
* succeed, records accumulate, nothing errors. This reads the daemon log and reports
|
|
8
|
+
* per-model rates so a regression is visible instead of inferred.
|
|
9
|
+
*
|
|
10
|
+
* Usage: node scripts/peon-operations-watch.mjs [--since 2026-09-09] [--json]
|
|
11
|
+
*/
|
|
12
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
|
|
16
|
+
const LOG_DIR = join(homedir(), "Library", "Logs", "Peon");
|
|
17
|
+
const args = process.argv.slice(2);
|
|
18
|
+
const since = (args[args.indexOf("--since") + 1] || "").match(/^\d{4}-\d{2}-\d{2}/) ? args[args.indexOf("--since") + 1] : null;
|
|
19
|
+
const asJson = args.includes("--json");
|
|
20
|
+
|
|
21
|
+
const files = readdirSync(LOG_DIR).filter((f) => f.startsWith("daemon.jsonl")).map((f) => join(LOG_DIR, f));
|
|
22
|
+
const runs = [];
|
|
23
|
+
for (const file of files) {
|
|
24
|
+
let text = "";
|
|
25
|
+
try { text = readFileSync(file, "utf8"); } catch { continue; }
|
|
26
|
+
for (const line of text.split("\n")) {
|
|
27
|
+
if (!line.includes("process_finish")) continue;
|
|
28
|
+
let entry;
|
|
29
|
+
try { entry = JSON.parse(line); } catch { continue; }
|
|
30
|
+
if (!/^(auto_)?process_finish$/.test(entry.type ?? "")) continue;
|
|
31
|
+
if (entry.status !== "processed") continue;
|
|
32
|
+
if (since && (entry.createdAt ?? "") < since) continue;
|
|
33
|
+
runs.push(entry);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
runs.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
|
|
37
|
+
|
|
38
|
+
const byModel = new Map();
|
|
39
|
+
for (const r of runs) {
|
|
40
|
+
const key = r.model ?? "unknown";
|
|
41
|
+
const a = byModel.get(key) ?? { runs: 0, operationsEmitted: 0, superseded: 0, obsoleted: 0, recordsAdded: 0, merged: 0, first: r.createdAt, last: r.createdAt };
|
|
42
|
+
a.runs += 1;
|
|
43
|
+
for (const k of ["operationsEmitted", "superseded", "obsoleted", "recordsAdded", "merged"]) a[k] += Number(r[k] ?? 0);
|
|
44
|
+
a.last = r.createdAt;
|
|
45
|
+
byModel.set(key, a);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const report = [...byModel.entries()].map(([model, a]) => ({
|
|
49
|
+
model,
|
|
50
|
+
runs: a.runs,
|
|
51
|
+
opsPerRun: +(a.operationsEmitted / a.runs).toFixed(2),
|
|
52
|
+
supersededPerRun: +(a.superseded / a.runs).toFixed(2),
|
|
53
|
+
addedPerRun: +(a.recordsAdded / a.runs).toFixed(2),
|
|
54
|
+
mergedPerRun: +(a.merged / a.runs).toFixed(2),
|
|
55
|
+
totals: { ...a, first: undefined, last: undefined },
|
|
56
|
+
window: `${String(a.first).slice(0, 19)} .. ${String(a.last).slice(0, 19)}`
|
|
57
|
+
}));
|
|
58
|
+
|
|
59
|
+
if (asJson) { console.log(JSON.stringify({ generatedAt: new Date().toISOString(), since, report }, null, 2)); process.exit(0); }
|
|
60
|
+
|
|
61
|
+
console.log(`Peon consolidation quality — ${new Date().toISOString().slice(0, 19)}Z${since ? ` (since ${since})` : ""}`);
|
|
62
|
+
console.log(`total consolidations: ${runs.length}\n`);
|
|
63
|
+
console.log(" model runs ops/run superseded/run added/run merged/run");
|
|
64
|
+
for (const r of report.sort((a, b) => b.runs - a.runs)) {
|
|
65
|
+
console.log(
|
|
66
|
+
` ${r.model.padEnd(32).slice(0, 32)} ${String(r.runs).padStart(5)} ${String(r.opsPerRun).padStart(7)} ${String(r.supersededPerRun).padStart(14)} ${String(r.addedPerRun).padStart(9)} ${String(r.mergedPerRun).padStart(10)}`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const local = report.find((r) => !r.model.includes("/") && r.model !== "manual-ai-result" && r.model !== "unknown");
|
|
70
|
+
if (local) {
|
|
71
|
+
console.log();
|
|
72
|
+
if (local.runs < 10) {
|
|
73
|
+
console.log(` NOTE: only ${local.runs} run(s) on ${local.model} — too few to judge. Keep collecting.`);
|
|
74
|
+
} else if (local.opsPerRun === 0) {
|
|
75
|
+
console.log(` WARNING: ${local.model} has emitted ZERO supersede/obsolete operations across ${local.runs} runs.`);
|
|
76
|
+
console.log(` Beliefs are accumulating but never being revised — a filing cabinet, not a brain.`);
|
|
77
|
+
} else {
|
|
78
|
+
console.log(` ${local.model}: ${local.opsPerRun} ops/run, ${local.supersededPerRun} superseded/run.`);
|
|
79
|
+
}
|
|
80
|
+
}
|
package/dist/roundtable.d.ts
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
import type { ProjectContext } from "./types.js";
|
|
2
|
-
export type RoundtableAgent = "codex" | "claude";
|
|
3
|
-
export type RoundtableAgentMode = "chat" | "research" | "implement" | "review" | "verify";
|
|
4
|
-
export type RoundtableStage = "queued" | "discussion" | "proposals" | "critiques" | "synthesis" | "implementation" | "review" | "verification" | "complete" | "failed";
|
|
5
|
-
export interface RoundtableTurn {
|
|
6
|
-
id: string;
|
|
7
|
-
agent: RoundtableAgent | "joint" | "user";
|
|
8
|
-
kind: "discussion" | "proposal" | "critique" | "recommendation" | "implementation" | "review" | "verification";
|
|
9
|
-
content: string;
|
|
10
|
-
createdAt: string;
|
|
11
|
-
}
|
|
12
|
-
export interface RoundtableActivity {
|
|
13
|
-
id: string;
|
|
14
|
-
agent: RoundtableAgent | "room";
|
|
15
|
-
type: "status" | "code" | "test" | "review" | "result" | "artifact";
|
|
16
|
-
title: string;
|
|
17
|
-
detail?: string;
|
|
18
|
-
state: "running" | "passed" | "failed" | "info";
|
|
19
|
-
createdAt: string;
|
|
20
|
-
updatedAt: string;
|
|
21
|
-
}
|
|
22
|
-
export interface RoundtableRun {
|
|
23
|
-
id: string;
|
|
24
|
-
projectPath: string;
|
|
25
|
-
question: string;
|
|
26
|
-
mode: "autonomous" | "focused";
|
|
27
|
-
status: "queued" | "running" | "open" | "awaiting_approval" | "executing" | "complete" | "rejected" | "failed";
|
|
28
|
-
stage: RoundtableStage;
|
|
29
|
-
createdAt: string;
|
|
30
|
-
updatedAt: string;
|
|
31
|
-
turns: RoundtableTurn[];
|
|
32
|
-
activities?: RoundtableActivity[];
|
|
33
|
-
liveSummary?: string;
|
|
34
|
-
synthesizedBy?: RoundtableAgent;
|
|
35
|
-
approvedAt?: string;
|
|
36
|
-
error?: string;
|
|
37
|
-
}
|
|
38
|
-
export interface RoundtableAgentRunner {
|
|
39
|
-
run(agent: RoundtableAgent, prompt: string, projectPath: string, mode?: RoundtableAgentMode): Promise<string>;
|
|
40
|
-
availability(): Promise<Record<RoundtableAgent, {
|
|
41
|
-
available: boolean;
|
|
42
|
-
label: string;
|
|
43
|
-
}>>;
|
|
44
|
-
}
|
|
45
|
-
export interface RoundtableMemoryPort {
|
|
46
|
-
getContext(input: {
|
|
47
|
-
projectPath: string;
|
|
48
|
-
query?: string;
|
|
49
|
-
maxChars?: number;
|
|
50
|
-
}): Promise<ProjectContext>;
|
|
51
|
-
saveProposal(projectPath: string, question: string, proposal: string): Promise<void>;
|
|
52
|
-
saveDecision(projectPath: string, question: string, decision: string): Promise<void>;
|
|
53
|
-
saveResult(projectPath: string, question: string, result: string): Promise<void>;
|
|
54
|
-
}
|
|
55
|
-
export interface RoundtableManagerOptions {
|
|
56
|
-
stateDir: string;
|
|
57
|
-
memory: RoundtableMemoryPort;
|
|
58
|
-
runner?: RoundtableAgentRunner;
|
|
59
|
-
now?: () => string;
|
|
60
|
-
}
|
|
61
|
-
export declare class RoundtableManager {
|
|
62
|
-
private readonly options;
|
|
63
|
-
private readonly runs;
|
|
64
|
-
private readonly subscribers;
|
|
65
|
-
private readonly runner;
|
|
66
|
-
private readonly now;
|
|
67
|
-
private queue;
|
|
68
|
-
private persistQueue;
|
|
69
|
-
constructor(options: RoundtableManagerOptions);
|
|
70
|
-
initialize(): Promise<void>;
|
|
71
|
-
availability(): Promise<Record<RoundtableAgent, {
|
|
72
|
-
available: boolean;
|
|
73
|
-
label: string;
|
|
74
|
-
}>>;
|
|
75
|
-
list(projectPath?: string): RoundtableRun[];
|
|
76
|
-
get(id: string): RoundtableRun | undefined;
|
|
77
|
-
subscribe(listener: (run: RoundtableRun) => void): () => void;
|
|
78
|
-
start(projectPath: string, rawQuestion?: string): Promise<RoundtableRun>;
|
|
79
|
-
approve(id: string): Promise<RoundtableRun>;
|
|
80
|
-
message(id: string, rawMessage: unknown): Promise<RoundtableRun>;
|
|
81
|
-
propose(id: string): Promise<RoundtableRun>;
|
|
82
|
-
continueDiscussion(id: string): Promise<RoundtableRun>;
|
|
83
|
-
reject(id: string): Promise<RoundtableRun>;
|
|
84
|
-
private execute;
|
|
85
|
-
private executeApproved;
|
|
86
|
-
private startActivity;
|
|
87
|
-
private addActivity;
|
|
88
|
-
private finishActivity;
|
|
89
|
-
private failRunningActivities;
|
|
90
|
-
private concludeAgreement;
|
|
91
|
-
private executeRevision;
|
|
92
|
-
private executeCasualReply;
|
|
93
|
-
private executeProposal;
|
|
94
|
-
private touch;
|
|
95
|
-
private emit;
|
|
96
|
-
private persist;
|
|
97
|
-
}
|
|
98
|
-
export declare class LocalSubscriptionRunner implements RoundtableAgentRunner {
|
|
99
|
-
availability(): Promise<Record<RoundtableAgent, {
|
|
100
|
-
available: boolean;
|
|
101
|
-
label: string;
|
|
102
|
-
}>>;
|
|
103
|
-
run(agent: RoundtableAgent, prompt: string, projectPath: string, mode?: RoundtableAgentMode): Promise<string>;
|
|
104
|
-
}
|