peon-mem 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +301 -0
- package/bin/peon-mem.mjs +273 -0
- package/dist/brain.d.ts +72 -0
- package/dist/brain.js +224 -0
- package/dist/compression.d.ts +9 -0
- package/dist/compression.js +37 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.js +99 -0
- package/dist/daemon-cli.d.ts +2 -0
- package/dist/daemon-cli.js +54 -0
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +1078 -0
- package/dist/embedding-store.d.ts +43 -0
- package/dist/embedding-store.js +169 -0
- package/dist/embeddings.d.ts +93 -0
- package/dist/embeddings.js +345 -0
- package/dist/entities.d.ts +61 -0
- package/dist/entities.js +191 -0
- package/dist/entity-extraction.d.ts +33 -0
- package/dist/entity-extraction.js +75 -0
- package/dist/eval-metrics.d.ts +27 -0
- package/dist/eval-metrics.js +50 -0
- package/dist/evaluation.d.ts +58 -0
- package/dist/evaluation.js +244 -0
- package/dist/global-extraction.d.ts +15 -0
- package/dist/global-extraction.js +61 -0
- package/dist/global-memory.d.ts +43 -0
- package/dist/global-memory.js +306 -0
- package/dist/global-promotion.d.ts +25 -0
- package/dist/global-promotion.js +29 -0
- package/dist/hyde.d.ts +31 -0
- package/dist/hyde.js +46 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +246 -0
- package/dist/injection.d.ts +38 -0
- package/dist/injection.js +133 -0
- package/dist/logger.d.ts +17 -0
- package/dist/logger.js +63 -0
- package/dist/memory-mutations.d.ts +24 -0
- package/dist/memory-mutations.js +57 -0
- package/dist/memory-store.d.ts +194 -0
- package/dist/memory-store.js +1205 -0
- package/dist/monitor.d.ts +13 -0
- package/dist/monitor.js +977 -0
- package/dist/overview.d.ts +73 -0
- package/dist/overview.js +104 -0
- package/dist/processor.d.ts +90 -0
- package/dist/processor.js +450 -0
- package/dist/quality.d.ts +86 -0
- package/dist/quality.js +338 -0
- package/dist/recuration.d.ts +13 -0
- package/dist/recuration.js +65 -0
- package/dist/reranker.d.ts +34 -0
- package/dist/reranker.js +89 -0
- package/dist/retrieval.d.ts +106 -0
- package/dist/retrieval.js +392 -0
- package/dist/session-index.d.ts +34 -0
- package/dist/session-index.js +87 -0
- package/dist/temporal.d.ts +20 -0
- package/dist/temporal.js +62 -0
- package/dist/token-ab-monitor.d.ts +1 -0
- package/dist/token-ab-monitor.js +7 -0
- package/dist/tools.d.ts +232 -0
- package/dist/tools.js +546 -0
- package/dist/types.d.ts +169 -0
- package/dist/types.js +1 -0
- package/docs/assets/neural-universe.png +0 -0
- package/package.json +57 -0
- package/scripts/claude-peon-hook.mjs +522 -0
- package/scripts/codex-peon-hook.mjs +4 -0
- package/scripts/eval-retrieval-labeled.mjs +135 -0
- package/scripts/eval-retrieval.mjs +96 -0
- package/scripts/evaluate-peon.mjs +47 -0
- package/scripts/install-peon-stl.mjs +82 -0
- package/scripts/install-peon.mjs +318 -0
- package/scripts/lib/eval-ledger.mjs +104 -0
- package/scripts/lib/stl-classify.mjs +44 -0
- package/scripts/longmemeval-eval.mjs +144 -0
- package/scripts/peon-report.mjs +155 -0
- package/scripts/peon-stl.mjs +506 -0
- package/scripts/token-ab-monitor.html +235 -0
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
import { cosineSimilarity } from "./embeddings.js";
|
|
2
|
+
import { canonicalizeEntity } from "./entities.js";
|
|
3
|
+
const DEFAULT_MIN_SIMILARITY = 0.15;
|
|
4
|
+
const stopWords = new Set([
|
|
5
|
+
"a",
|
|
6
|
+
"an",
|
|
7
|
+
"and",
|
|
8
|
+
"are",
|
|
9
|
+
"as",
|
|
10
|
+
"for",
|
|
11
|
+
"in",
|
|
12
|
+
"is",
|
|
13
|
+
"of",
|
|
14
|
+
"on",
|
|
15
|
+
"or",
|
|
16
|
+
"the",
|
|
17
|
+
"to",
|
|
18
|
+
"use",
|
|
19
|
+
"with"
|
|
20
|
+
]);
|
|
21
|
+
const RRF_K = 60; // standard reciprocal-rank-fusion constant
|
|
22
|
+
const PIN_BOOST = 0.05;
|
|
23
|
+
// Associative graph signal weight (fraction of a direct RRF signal), for the OPT-IN expandGraph
|
|
24
|
+
// path only. Default 0.1: the labeled eval showed higher weights hurt top-K relevance (0.5 →
|
|
25
|
+
// −2.9% Recall@10, 1.0 → −3.8%) while ~0.1 is neutral. Env-tunable (PEON_GRAPH_WEIGHT) for sweeps.
|
|
26
|
+
const GRAPH_FUSION_WEIGHT = (() => {
|
|
27
|
+
const env = process.env.PEON_GRAPH_WEIGHT;
|
|
28
|
+
return env !== undefined && Number.isFinite(Number(env)) ? Number(env) : 0.1;
|
|
29
|
+
})();
|
|
30
|
+
// Type priority is a PRIOR vote in the fusion (durable types over ephemeral) — a
|
|
31
|
+
// rank, not a hand-tuned weight, so it stays robust alongside the other signals.
|
|
32
|
+
const TYPE_PRIORITY = {
|
|
33
|
+
decision: 6, preference: 5, fact: 4, artifact: 3, summary: 2, open_question: 1, timeline: 0
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Hybrid retrieval by Reciprocal Rank Fusion. Each signal (lexical, semantic,
|
|
37
|
+
* quality, recency, reinforcement strength) ranks the relevance-gated candidates
|
|
38
|
+
* independently; we fuse by Σ 1/(k+rank). RRF is weight-free — it removes the
|
|
39
|
+
* fragile hand-tuned score constants by comparing RANKS, which are commensurable
|
|
40
|
+
* across signals where raw scores are not. Pinned beliefs are boosted; archived/
|
|
41
|
+
* superseded are excluded by default (they are the long-term tier, not working memory).
|
|
42
|
+
*/
|
|
43
|
+
export function rankMemoryRecords(records, query, options = {}) {
|
|
44
|
+
const terms = queryTerms(query);
|
|
45
|
+
const fileTerms = queryFileTerms(query);
|
|
46
|
+
const nowMs = timestamp(options.now ?? Date.now());
|
|
47
|
+
const limit = Math.max(0, Math.trunc(options.limit ?? 50));
|
|
48
|
+
const semantic = options.semantic;
|
|
49
|
+
const minSimilarity = semantic?.minSimilarity ?? DEFAULT_MIN_SIMILARITY;
|
|
50
|
+
const hasQuery = terms.length > 0;
|
|
51
|
+
const signals = records.map((record, index) => {
|
|
52
|
+
const reasons = [];
|
|
53
|
+
const lexical = lexicalScore(record, terms, fileTerms, reasons);
|
|
54
|
+
let semSim = 0;
|
|
55
|
+
if (semantic && semantic.queryVector.length > 0) {
|
|
56
|
+
const vector = semantic.vectorById.get(record.id);
|
|
57
|
+
if (vector && vector.length > 0) {
|
|
58
|
+
const sim = cosineSimilarity(semantic.queryVector, vector);
|
|
59
|
+
if (sim >= minSimilarity) {
|
|
60
|
+
semSim = sim;
|
|
61
|
+
reasons.push({ kind: "semantic", label: `semantic match ${sim.toFixed(2)}`, score: roundScore(sim) });
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const quality = clamp(record.score.importance) * 0.6 + clamp(record.score.confidence) * 0.4;
|
|
66
|
+
const strength = clamp(typeof record.strength === "number" ? record.strength : record.score.importance)
|
|
67
|
+
+ Math.min(0.5, Math.log1p(record.recallCount ?? 0) / 6); // reinforcement bonus
|
|
68
|
+
const graph = options.graphActivation?.get(record.id) ?? 0;
|
|
69
|
+
if (graph > 0 && lexical === 0 && semSim === 0) {
|
|
70
|
+
reasons.push({ kind: "entity", label: "linked via entity graph", score: roundScore(graph) });
|
|
71
|
+
}
|
|
72
|
+
return { record, index, lexical, semantic: semSim, quality, recency: recencyScore(record, nowMs), strength, graph, reasons };
|
|
73
|
+
});
|
|
74
|
+
// Relevance gate: with a query, keep only beliefs that matched lexically or semantically.
|
|
75
|
+
// Tier filtering (active vs archived/superseded) is intentionally NOT done here — the
|
|
76
|
+
// injection/context layer owns that, so it can both record WHY a belief was omitted and
|
|
77
|
+
// recover history on demand (includeInactive). The ranker stays a pure relevance ranker.
|
|
78
|
+
const gated = signals.filter((s) => {
|
|
79
|
+
if (!hasQuery)
|
|
80
|
+
return true;
|
|
81
|
+
// Admit a belief that matched the query OR that the entity graph activated (associative recall).
|
|
82
|
+
return s.lexical > 0 || s.semantic > 0 || s.graph > 0;
|
|
83
|
+
});
|
|
84
|
+
if (gated.length === 0)
|
|
85
|
+
return [];
|
|
86
|
+
// Build per-signal rank maps over the gated candidates.
|
|
87
|
+
const lexRank = rankMap(gated, (s) => s.lexical, true);
|
|
88
|
+
const semRank = rankMap(gated, (s) => s.semantic, true);
|
|
89
|
+
const qualRank = rankMap(gated, (s) => s.quality, false);
|
|
90
|
+
const recRank = rankMap(gated, (s) => s.recency, false);
|
|
91
|
+
const strRank = rankMap(gated, (s) => s.strength, false);
|
|
92
|
+
const typeRank = rankMap(gated, (s) => TYPE_PRIORITY[s.record.type] ?? 0, false);
|
|
93
|
+
const graphRank = rankMap(gated, (s) => s.graph, true);
|
|
94
|
+
const ranked = gated.map((s) => {
|
|
95
|
+
let score = 0;
|
|
96
|
+
if (s.lexical > 0)
|
|
97
|
+
score += 1 / (RRF_K + lexRank(s));
|
|
98
|
+
if (s.semantic > 0)
|
|
99
|
+
score += 1 / (RRF_K + semRank(s));
|
|
100
|
+
// Associative graph signal — DAMPED (×GRAPH_FUSION_WEIGHT) so it's secondary to direct
|
|
101
|
+
// matches: a strong association can enter the top-K and displace a weak direct hit, but
|
|
102
|
+
// won't outrank genuine query matches.
|
|
103
|
+
if (s.graph > 0)
|
|
104
|
+
score += GRAPH_FUSION_WEIGHT * (1 / (RRF_K + graphRank(s)));
|
|
105
|
+
// Priors always participate (relevance already gated the candidate set).
|
|
106
|
+
score += 1 / (RRF_K + qualRank(s));
|
|
107
|
+
score += 1 / (RRF_K + recRank(s));
|
|
108
|
+
score += 1 / (RRF_K + strRank(s));
|
|
109
|
+
score += 1 / (RRF_K + typeRank(s));
|
|
110
|
+
if (s.record.pinned)
|
|
111
|
+
score += PIN_BOOST;
|
|
112
|
+
if (s.record.status === "stale")
|
|
113
|
+
score -= 0.01;
|
|
114
|
+
if (s.record.status === "conflicted")
|
|
115
|
+
score -= 0.005;
|
|
116
|
+
const reasons = [...s.reasons,
|
|
117
|
+
{ kind: "quality", label: `importance ${clamp(s.record.score.importance).toFixed(2)}`, score: 1 / (RRF_K + qualRank(s)) },
|
|
118
|
+
{ kind: "recency", label: `updated ${s.record.updatedAt}`, score: 1 / (RRF_K + recRank(s)) }];
|
|
119
|
+
if ((s.record.recallCount ?? 0) > 0)
|
|
120
|
+
reasons.push({ kind: "status", label: `recalled ${s.record.recallCount}×`, score: 1 / (RRF_K + strRank(s)) });
|
|
121
|
+
return {
|
|
122
|
+
record: s.record,
|
|
123
|
+
score: roundScore(score),
|
|
124
|
+
reasons,
|
|
125
|
+
explanation: s.reasons.map((r) => r.label).join("; "),
|
|
126
|
+
index: s.index
|
|
127
|
+
};
|
|
128
|
+
});
|
|
129
|
+
return ranked
|
|
130
|
+
.sort((left, right) => {
|
|
131
|
+
if (right.score !== left.score)
|
|
132
|
+
return right.score - left.score;
|
|
133
|
+
const updatedDelta = timestamp(right.record.updatedAt) - timestamp(left.record.updatedAt);
|
|
134
|
+
if (updatedDelta !== 0)
|
|
135
|
+
return updatedDelta;
|
|
136
|
+
return left.index - right.index;
|
|
137
|
+
})
|
|
138
|
+
.slice(0, limit)
|
|
139
|
+
.map(({ index: _index, ...item }) => item);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Build a total rank lookup (1-based, descending by key) using COMPETITION ranking:
|
|
143
|
+
* equal values share the same rank, so ties contribute equally to both records and
|
|
144
|
+
* the outcome is decided by the signals that actually differ (no arbitrary tie-break).
|
|
145
|
+
* Missing (gated-out) → large rank (~0 contribution).
|
|
146
|
+
*/
|
|
147
|
+
function rankMap(items, key, gateZero) {
|
|
148
|
+
const sorted = items
|
|
149
|
+
.map((s) => ({ s, v: key(s) }))
|
|
150
|
+
.filter((x) => (gateZero ? x.v > 0 : true))
|
|
151
|
+
.sort((a, b) => b.v - a.v);
|
|
152
|
+
const map = new Map();
|
|
153
|
+
let rank = 0;
|
|
154
|
+
let prev;
|
|
155
|
+
sorted.forEach((x, i) => {
|
|
156
|
+
if (prev === undefined || x.v !== prev)
|
|
157
|
+
rank = i + 1; // ties keep the prior rank
|
|
158
|
+
prev = x.v;
|
|
159
|
+
map.set(x.s, rank);
|
|
160
|
+
});
|
|
161
|
+
const last = sorted.length + items.length + 1;
|
|
162
|
+
return (s) => map.get(s) ?? last;
|
|
163
|
+
}
|
|
164
|
+
function lexicalScore(record, terms, fileTerms, reasons) {
|
|
165
|
+
const text = searchableText(record);
|
|
166
|
+
let score = 0;
|
|
167
|
+
const matchedTerms = terms.filter((term) => text.includes(term));
|
|
168
|
+
if (matchedTerms.length > 0) {
|
|
169
|
+
score += matchedTerms.length * 2;
|
|
170
|
+
reasons.push({ kind: "query_term", label: `matched query terms ${matchedTerms.join(", ")}`, score: matchedTerms.length * 2 });
|
|
171
|
+
}
|
|
172
|
+
const matchedEntities = record.entities.filter((entity) => {
|
|
173
|
+
const e = entity.toLowerCase();
|
|
174
|
+
return terms.some((term) => e.includes(term) || term.includes(e));
|
|
175
|
+
});
|
|
176
|
+
if (matchedEntities.length > 0) {
|
|
177
|
+
score += matchedEntities.length * 3;
|
|
178
|
+
reasons.push({ kind: "entity", label: `matched entity ${matchedEntities.join(", ")}`, score: matchedEntities.length * 3 });
|
|
179
|
+
}
|
|
180
|
+
const matchedFiles = record.entities.filter((entity) => isFileLike(entity) && fileTerms.some((term) => entity.toLowerCase().includes(term)));
|
|
181
|
+
if (matchedFiles.length > 0) {
|
|
182
|
+
score += matchedFiles.length * 4;
|
|
183
|
+
reasons.push({ kind: "file", label: `matched file ${matchedFiles.join(", ")}`, score: matchedFiles.length * 4 });
|
|
184
|
+
}
|
|
185
|
+
return score;
|
|
186
|
+
}
|
|
187
|
+
export function selectMemoryRecordsForContext(rankedRecords, options) {
|
|
188
|
+
const maxChars = Math.max(0, Math.trunc(options.maxChars));
|
|
189
|
+
const formatter = options.recordFormatter ?? defaultRecordFormatter;
|
|
190
|
+
const records = [];
|
|
191
|
+
const omitted = [];
|
|
192
|
+
let totalChars = 0;
|
|
193
|
+
for (const item of rankedRecords) {
|
|
194
|
+
const length = formatter(item).length;
|
|
195
|
+
if (length <= maxChars - totalChars) {
|
|
196
|
+
records.push(item);
|
|
197
|
+
totalChars += length;
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
omitted.push(item);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return { records, omitted, totalChars, maxChars };
|
|
204
|
+
}
|
|
205
|
+
/** Default trade-off: 0.7 weight on relevance, 0.3 on novelty. Tuned for coverage without losing the top hit. */
|
|
206
|
+
export const DEFAULT_MMR_LAMBDA = 0.7;
|
|
207
|
+
/**
|
|
208
|
+
* Re-order ranked records by Maximal Marginal Relevance so the injected block has
|
|
209
|
+
* COVERAGE instead of five paraphrases of the same belief. Each pick maximizes
|
|
210
|
+
* λ·relevance − (1−λ)·maxSimilarity(candidate, alreadyPicked)
|
|
211
|
+
* Relevance is the record's existing fused score (max-normalized to [0,1] within the
|
|
212
|
+
* set); similarity is lexical Jaccard over content+entity tokens — cheap, deterministic,
|
|
213
|
+
* and embedding-free so it works in pure local-first mode. The single most relevant
|
|
214
|
+
* record is always selected first, so the top hit is never displaced by diversification.
|
|
215
|
+
*/
|
|
216
|
+
export function diversifyByMMR(records, lambda = DEFAULT_MMR_LAMBDA) {
|
|
217
|
+
if (records.length <= 2)
|
|
218
|
+
return [...records];
|
|
219
|
+
const λ = Math.max(0, Math.min(1, lambda));
|
|
220
|
+
const maxScore = Math.max(...records.map((r) => r.score), 0);
|
|
221
|
+
const rel = (item) => (maxScore > 0 ? item.score / maxScore : 0);
|
|
222
|
+
const tokens = records.map((r) => recordTokens(r.record));
|
|
223
|
+
const remaining = records.map((_, i) => i);
|
|
224
|
+
const selected = [];
|
|
225
|
+
// Seed with the single most relevant record (input is already relevance-sorted).
|
|
226
|
+
let best = remaining[0];
|
|
227
|
+
for (const i of remaining)
|
|
228
|
+
if (rel(records[i]) > rel(records[best]))
|
|
229
|
+
best = i;
|
|
230
|
+
selected.push(best);
|
|
231
|
+
remaining.splice(remaining.indexOf(best), 1);
|
|
232
|
+
while (remaining.length > 0) {
|
|
233
|
+
let pick = remaining[0];
|
|
234
|
+
let pickScore = -Infinity;
|
|
235
|
+
for (const i of remaining) {
|
|
236
|
+
let maxSim = 0;
|
|
237
|
+
for (const s of selected) {
|
|
238
|
+
const sim = jaccard(tokens[i], tokens[s]);
|
|
239
|
+
if (sim > maxSim)
|
|
240
|
+
maxSim = sim;
|
|
241
|
+
}
|
|
242
|
+
const mmr = λ * rel(records[i]) - (1 - λ) * maxSim;
|
|
243
|
+
if (mmr > pickScore) {
|
|
244
|
+
pickScore = mmr;
|
|
245
|
+
pick = i;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
selected.push(pick);
|
|
249
|
+
remaining.splice(remaining.indexOf(pick), 1);
|
|
250
|
+
}
|
|
251
|
+
return selected.map((i) => records[i]);
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Entity-graph spreading activation (the associative-recall layer). Lexical/semantic ranking
|
|
255
|
+
* finds beliefs that match the QUERY; this finds beliefs in the ANSWER's neighbourhood by
|
|
256
|
+
* spreading activation from the top hits through shared entities, with three brain-like rules
|
|
257
|
+
* the old flat 1-hop expander lacked:
|
|
258
|
+
* - DISTANCE DECAY — a global damping (λ) keeps neighbours below direct matches.
|
|
259
|
+
* - MULTI-SOURCE SUMMATION — a belief lit through several shared entities (or several seeds)
|
|
260
|
+
* accumulates activation, so it outranks one lit through a single weak link.
|
|
261
|
+
* - HUB DAMPING — rare entities transmit more activation (1/log₂(2+degree)); super-hubs
|
|
262
|
+
* (e.g. a file mentioned by 80 beliefs) are skipped so the graph isn't a hairball.
|
|
263
|
+
* Domain entities (people/papers/concepts) spread more than code entities. Pure + deterministic.
|
|
264
|
+
* Neighbours come back with small scores in their own band and are meant to be appended AFTER
|
|
265
|
+
* the direct results, never displacing them.
|
|
266
|
+
*/
|
|
267
|
+
/**
|
|
268
|
+
* Raw spreading activation: id → accumulated activation for beliefs in the seeds' entity
|
|
269
|
+
* neighbourhood (excluding the seeds themselves). The associative substrate, shared by the
|
|
270
|
+
* fused-ranking path (passed as RetrievalOptions.graphActivation) and the legacy append path
|
|
271
|
+
* (expandByEntityGraph). Excludes seed ids so direct hits aren't double-counted.
|
|
272
|
+
*/
|
|
273
|
+
export function computeGraphActivation(seeds, pool, options = {}) {
|
|
274
|
+
const seedDepth = Math.max(1, Math.trunc(options.seedDepth ?? 8));
|
|
275
|
+
const damping = options.damping ?? 0.5;
|
|
276
|
+
const codeWeight = options.codeWeight ?? 0.4;
|
|
277
|
+
const hubDegreeCap = Math.max(1, Math.trunc(options.hubDegreeCap ?? 40));
|
|
278
|
+
const active = pool.filter((record) => record.status === "active");
|
|
279
|
+
const entityToRecords = new Map();
|
|
280
|
+
for (const record of active) {
|
|
281
|
+
for (const e of unique(record.entities)) {
|
|
282
|
+
const list = entityToRecords.get(e) ?? entityToRecords.set(e, []).get(e);
|
|
283
|
+
list.push(record);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
const seedIds = new Set(seeds.map((s) => s.record.id));
|
|
287
|
+
const topSeeds = seeds.slice(0, seedDepth);
|
|
288
|
+
const activation = new Map();
|
|
289
|
+
if (topSeeds.length === 0)
|
|
290
|
+
return activation;
|
|
291
|
+
const maxSeedScore = Math.max(...topSeeds.map((s) => s.score), 1e-9);
|
|
292
|
+
topSeeds.forEach((seed, i) => {
|
|
293
|
+
const seedActivation = (seed.score > 0 ? seed.score / maxSeedScore : 0) / (1 + i * 0.3); // rank decay
|
|
294
|
+
if (seedActivation <= 0)
|
|
295
|
+
return;
|
|
296
|
+
for (const e of unique(seed.record.entities)) {
|
|
297
|
+
const holders = entityToRecords.get(e);
|
|
298
|
+
if (!holders || holders.length > hubDegreeCap)
|
|
299
|
+
continue; // skip super-hubs
|
|
300
|
+
const ns = canonicalizeEntity(e)?.namespace ?? "code";
|
|
301
|
+
const entityWeight = (ns === "domain" ? 1 : codeWeight) / Math.log2(2 + holders.length);
|
|
302
|
+
for (const neighbor of holders) {
|
|
303
|
+
if (seedIds.has(neighbor.id))
|
|
304
|
+
continue; // never re-rank a direct hit
|
|
305
|
+
activation.set(neighbor.id, (activation.get(neighbor.id) ?? 0) + damping * seedActivation * entityWeight);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
return activation;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Entity-graph spreading activation, formatted as standalone neighbour records (legacy append
|
|
313
|
+
* path + the unit tests). The fused-ranking path uses computeGraphActivation directly via
|
|
314
|
+
* rankMemoryRecords' graphActivation option, which lets associations compete inside the top-K.
|
|
315
|
+
*/
|
|
316
|
+
export function expandByEntityGraph(seeds, pool, options = {}) {
|
|
317
|
+
const maxNeighbors = Math.max(0, Math.trunc(options.maxNeighbors ?? 6));
|
|
318
|
+
if (maxNeighbors === 0)
|
|
319
|
+
return [];
|
|
320
|
+
const activation = computeGraphActivation(seeds, pool, options);
|
|
321
|
+
if (activation.size === 0)
|
|
322
|
+
return [];
|
|
323
|
+
const recordById = new Map(pool.filter((r) => r.status === "active").map((r) => [r.id, r]));
|
|
324
|
+
const maxAct = Math.max(...activation.values());
|
|
325
|
+
if (!(maxAct > 0))
|
|
326
|
+
return []; // all-zero activation (e.g. damping/codeWeight 0) → avoid NaN scores
|
|
327
|
+
return [...activation.entries()]
|
|
328
|
+
.sort((a, b) => b[1] - a[1])
|
|
329
|
+
.slice(0, maxNeighbors)
|
|
330
|
+
.map(([id, act]) => {
|
|
331
|
+
const record = recordById.get(id);
|
|
332
|
+
const label = "linked via entity graph";
|
|
333
|
+
const score = roundScore(0.02 * (act / maxAct)); // small supplementary band
|
|
334
|
+
return { record, score, reasons: [{ kind: "entity", label, score }], explanation: label };
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
function recordTokens(record) {
|
|
338
|
+
const raw = `${record.normalized} ${record.entities.join(" ")}`.toLowerCase();
|
|
339
|
+
return new Set(raw
|
|
340
|
+
.split(/[^a-z0-9_.\/-]+/)
|
|
341
|
+
.map((t) => t.trim())
|
|
342
|
+
.filter((t) => t.length > 1 && !stopWords.has(t)));
|
|
343
|
+
}
|
|
344
|
+
function jaccard(a, b) {
|
|
345
|
+
if (a.size === 0 || b.size === 0)
|
|
346
|
+
return 0;
|
|
347
|
+
let inter = 0;
|
|
348
|
+
for (const t of a)
|
|
349
|
+
if (b.has(t))
|
|
350
|
+
inter++;
|
|
351
|
+
const union = a.size + b.size - inter;
|
|
352
|
+
return union > 0 ? inter / union : 0;
|
|
353
|
+
}
|
|
354
|
+
function defaultRecordFormatter(item) {
|
|
355
|
+
return `- [${item.record.type}] ${item.record.content}\n`;
|
|
356
|
+
}
|
|
357
|
+
function queryTerms(query) {
|
|
358
|
+
return unique((query ?? "")
|
|
359
|
+
.toLowerCase()
|
|
360
|
+
.split(/[^a-z0-9_.\/-]+/)
|
|
361
|
+
.map((term) => term.trim())
|
|
362
|
+
.filter((term) => term.length > 1 && !stopWords.has(term)));
|
|
363
|
+
}
|
|
364
|
+
function queryFileTerms(query) {
|
|
365
|
+
return queryTerms(query).filter(isFileLike);
|
|
366
|
+
}
|
|
367
|
+
function searchableText(record) {
|
|
368
|
+
return `${record.type} ${record.content} ${record.normalized} ${record.entities.join(" ")}`.toLowerCase();
|
|
369
|
+
}
|
|
370
|
+
function recencyScore(record, nowMs) {
|
|
371
|
+
const updatedMs = timestamp(record.updatedAt || record.createdAt);
|
|
372
|
+
if (!Number.isFinite(updatedMs) || !Number.isFinite(nowMs))
|
|
373
|
+
return 0;
|
|
374
|
+
const ageDays = Math.max(0, (nowMs - updatedMs) / 86_400_000);
|
|
375
|
+
return roundScore(1 / (1 + ageDays / 30));
|
|
376
|
+
}
|
|
377
|
+
function timestamp(value) {
|
|
378
|
+
const time = value instanceof Date ? value.getTime() : new Date(value).getTime();
|
|
379
|
+
return Number.isFinite(time) ? time : 0;
|
|
380
|
+
}
|
|
381
|
+
function isFileLike(value) {
|
|
382
|
+
return /[/.\\]/.test(value) || /\.[a-z0-9]+$/i.test(value);
|
|
383
|
+
}
|
|
384
|
+
function unique(values) {
|
|
385
|
+
return [...new Set(values)];
|
|
386
|
+
}
|
|
387
|
+
function clamp(value) {
|
|
388
|
+
return Math.max(0, Math.min(1, Number.isFinite(value) ? value : 0));
|
|
389
|
+
}
|
|
390
|
+
function roundScore(value) {
|
|
391
|
+
return Math.round(value * 1000) / 1000;
|
|
392
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable sessionId → project mapping.
|
|
3
|
+
*
|
|
4
|
+
* The in-process tools layer needs to resolve which project a sessionId belongs
|
|
5
|
+
* to. Holding that only in memory means a daemon restart mid-session orphans
|
|
6
|
+
* every in-flight session ("Unknown Peon session"). This index persists the
|
|
7
|
+
* mapping to disk so sessions survive restarts.
|
|
8
|
+
*
|
|
9
|
+
* Writes are serialized through a queue; each write rewrites the full JSON map
|
|
10
|
+
* (fine for a local single-user tool). Ended sessions are removed to keep the
|
|
11
|
+
* file small; a crash may leave a stale "active" entry, which `prune` clears.
|
|
12
|
+
*/
|
|
13
|
+
export interface SessionIndexRecord {
|
|
14
|
+
sessionId: string;
|
|
15
|
+
projectPath: string;
|
|
16
|
+
client: string;
|
|
17
|
+
cwd: string;
|
|
18
|
+
startedAt: string;
|
|
19
|
+
}
|
|
20
|
+
export declare function defaultSessionIndexPath(): string;
|
|
21
|
+
export declare class SessionIndex {
|
|
22
|
+
private readonly filePath;
|
|
23
|
+
private cache?;
|
|
24
|
+
private writeQueue;
|
|
25
|
+
constructor(filePath?: string);
|
|
26
|
+
get(sessionId: string): Promise<SessionIndexRecord | undefined>;
|
|
27
|
+
set(record: SessionIndexRecord): Promise<void>;
|
|
28
|
+
remove(sessionId: string): Promise<void>;
|
|
29
|
+
active(): Promise<SessionIndexRecord[]>;
|
|
30
|
+
/** Drop sessions older than maxAgeMs (stale entries from crashed runs). */
|
|
31
|
+
prune(now: number, maxAgeMs: number): Promise<number>;
|
|
32
|
+
private load;
|
|
33
|
+
private persist;
|
|
34
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
export function defaultSessionIndexPath() {
|
|
5
|
+
return join(homedir(), "Library", "Application Support", "Peon", "sessions-index.json");
|
|
6
|
+
}
|
|
7
|
+
export class SessionIndex {
|
|
8
|
+
filePath;
|
|
9
|
+
cache;
|
|
10
|
+
writeQueue = Promise.resolve();
|
|
11
|
+
constructor(filePath = defaultSessionIndexPath()) {
|
|
12
|
+
this.filePath = filePath;
|
|
13
|
+
}
|
|
14
|
+
async get(sessionId) {
|
|
15
|
+
return (await this.load()).get(sessionId);
|
|
16
|
+
}
|
|
17
|
+
async set(record) {
|
|
18
|
+
const map = await this.load();
|
|
19
|
+
map.set(record.sessionId, record);
|
|
20
|
+
await this.persist(map);
|
|
21
|
+
}
|
|
22
|
+
async remove(sessionId) {
|
|
23
|
+
const map = await this.load();
|
|
24
|
+
if (map.delete(sessionId))
|
|
25
|
+
await this.persist(map);
|
|
26
|
+
}
|
|
27
|
+
async active() {
|
|
28
|
+
return [...(await this.load()).values()];
|
|
29
|
+
}
|
|
30
|
+
/** Drop sessions older than maxAgeMs (stale entries from crashed runs). */
|
|
31
|
+
async prune(now, maxAgeMs) {
|
|
32
|
+
const map = await this.load();
|
|
33
|
+
let removed = 0;
|
|
34
|
+
for (const [id, record] of map) {
|
|
35
|
+
const started = Date.parse(record.startedAt);
|
|
36
|
+
if (Number.isFinite(started) && now - started > maxAgeMs) {
|
|
37
|
+
map.delete(id);
|
|
38
|
+
removed += 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (removed > 0)
|
|
42
|
+
await this.persist(map);
|
|
43
|
+
return removed;
|
|
44
|
+
}
|
|
45
|
+
async load() {
|
|
46
|
+
if (this.cache)
|
|
47
|
+
return this.cache;
|
|
48
|
+
const raw = await readFile(this.filePath, "utf8").catch(() => "");
|
|
49
|
+
const map = new Map();
|
|
50
|
+
if (raw.trim()) {
|
|
51
|
+
try {
|
|
52
|
+
const parsed = JSON.parse(raw);
|
|
53
|
+
if (parsed && typeof parsed === "object") {
|
|
54
|
+
for (const value of Object.values(parsed)) {
|
|
55
|
+
if (isSessionIndexRecord(value))
|
|
56
|
+
map.set(value.sessionId, value);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// Corrupt index → start clean rather than blocking all session resolution.
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
this.cache = map;
|
|
65
|
+
return map;
|
|
66
|
+
}
|
|
67
|
+
async persist(map) {
|
|
68
|
+
this.cache = map;
|
|
69
|
+
const snapshot = Object.fromEntries(map);
|
|
70
|
+
const write = async () => {
|
|
71
|
+
await mkdir(dirname(this.filePath), { recursive: true });
|
|
72
|
+
await writeFile(this.filePath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
|
73
|
+
};
|
|
74
|
+
this.writeQueue = this.writeQueue.then(write, write);
|
|
75
|
+
return this.writeQueue;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function isSessionIndexRecord(value) {
|
|
79
|
+
if (!value || typeof value !== "object")
|
|
80
|
+
return false;
|
|
81
|
+
const record = value;
|
|
82
|
+
return (typeof record.sessionId === "string" &&
|
|
83
|
+
typeof record.projectPath === "string" &&
|
|
84
|
+
typeof record.client === "string" &&
|
|
85
|
+
typeof record.cwd === "string" &&
|
|
86
|
+
typeof record.startedAt === "string");
|
|
87
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { MemoryRecord } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* The beliefs that were CURRENT as of `at`. A belief is current at `at` iff it had been created
|
|
4
|
+
* by then (createdAt ≤ at) and had not yet been retired/replaced by then — i.e. it is still live,
|
|
5
|
+
* or it was terminated (superseded/archived) only at a later time (updatedAt > at).
|
|
6
|
+
*/
|
|
7
|
+
export declare function currentAsOf(records: MemoryRecord[], at: string | number | Date): MemoryRecord[];
|
|
8
|
+
export type ChangeKind = "added" | "superseded" | "retired";
|
|
9
|
+
export interface ChangeEntry {
|
|
10
|
+
kind: ChangeKind;
|
|
11
|
+
at: string;
|
|
12
|
+
record: MemoryRecord;
|
|
13
|
+
replacementId?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The changelog over [from, to] (inclusive): beliefs added (created in window) and beliefs that
|
|
17
|
+
* stopped being current in the window — `superseded` (has a successor) or `retired` (obsoleted,
|
|
18
|
+
* no successor). Sorted chronologically. This is the "what changed" / knowledge-update view.
|
|
19
|
+
*/
|
|
20
|
+
export declare function changesBetween(records: MemoryRecord[], from: string | number | Date, to: string | number | Date): ChangeEntry[];
|
package/dist/temporal.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Temporal retrieval over the belief store. Peon already keeps the history of a belief: when a
|
|
3
|
+
* fact changes, the old record is flipped to `superseded` (its `updatedAt` is the moment it
|
|
4
|
+
* stopped being true) and linked to its successor via `supersededBy`; an obsoleted belief is
|
|
5
|
+
* superseded with no successor. That history lets us answer two questions a flat "current state"
|
|
6
|
+
* store cannot:
|
|
7
|
+
*
|
|
8
|
+
* • as-of — "what did we believe at time T?" (currentAsOf)
|
|
9
|
+
* • diff — "what changed between T1 and T2?" (changesBetween)
|
|
10
|
+
*
|
|
11
|
+
* Both are pure, deterministic, and timestamp-only — no LLM. They turn the supersession chain
|
|
12
|
+
* the consolidator already maintains into first-class time-travel queries.
|
|
13
|
+
*/
|
|
14
|
+
/** Statuses that mark a belief as no longer current; their `updatedAt` is when it stopped being current. */
|
|
15
|
+
const TERMINAL_STATUSES = new Set(["superseded", "archived"]);
|
|
16
|
+
function ms(value) {
|
|
17
|
+
const t = value instanceof Date ? value.getTime() : new Date(value).getTime();
|
|
18
|
+
return Number.isFinite(t) ? t : 0;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The beliefs that were CURRENT as of `at`. A belief is current at `at` iff it had been created
|
|
22
|
+
* by then (createdAt ≤ at) and had not yet been retired/replaced by then — i.e. it is still live,
|
|
23
|
+
* or it was terminated (superseded/archived) only at a later time (updatedAt > at).
|
|
24
|
+
*/
|
|
25
|
+
export function currentAsOf(records, at) {
|
|
26
|
+
const t = ms(at);
|
|
27
|
+
return records.filter((r) => {
|
|
28
|
+
if (ms(r.createdAt) > t)
|
|
29
|
+
return false; // didn't exist yet
|
|
30
|
+
if (TERMINAL_STATUSES.has(r.status))
|
|
31
|
+
return ms(r.updatedAt) > t; // retired only after `at`
|
|
32
|
+
return true; // active / stale / conflicted — live at `at`
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The changelog over [from, to] (inclusive): beliefs added (created in window) and beliefs that
|
|
37
|
+
* stopped being current in the window — `superseded` (has a successor) or `retired` (obsoleted,
|
|
38
|
+
* no successor). Sorted chronologically. This is the "what changed" / knowledge-update view.
|
|
39
|
+
*/
|
|
40
|
+
export function changesBetween(records, from, to) {
|
|
41
|
+
const lo = ms(from);
|
|
42
|
+
const hi = ms(to);
|
|
43
|
+
const entries = [];
|
|
44
|
+
for (const r of records) {
|
|
45
|
+
const created = ms(r.createdAt);
|
|
46
|
+
if (created >= lo && created <= hi) {
|
|
47
|
+
entries.push({ kind: "added", at: r.createdAt, record: r });
|
|
48
|
+
}
|
|
49
|
+
if (r.status === "superseded") {
|
|
50
|
+
const changedAt = ms(r.updatedAt);
|
|
51
|
+
if (changedAt >= lo && changedAt <= hi) {
|
|
52
|
+
entries.push({
|
|
53
|
+
kind: r.supersededBy ? "superseded" : "retired",
|
|
54
|
+
at: r.updatedAt,
|
|
55
|
+
record: r,
|
|
56
|
+
replacementId: r.supersededBy
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return entries.sort((a, b) => ms(a.at) - ms(b.at));
|
|
62
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function renderTokenAbMonitorHtml(): string;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join, dirname } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
5
|
+
export function renderTokenAbMonitorHtml() {
|
|
6
|
+
return readFileSync(join(__dirname, "..", "scripts", "token-ab-monitor.html"), "utf8");
|
|
7
|
+
}
|