peon-mem 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +301 -0
  3. package/bin/peon-mem.mjs +273 -0
  4. package/dist/brain.d.ts +72 -0
  5. package/dist/brain.js +224 -0
  6. package/dist/compression.d.ts +9 -0
  7. package/dist/compression.js +37 -0
  8. package/dist/config.d.ts +22 -0
  9. package/dist/config.js +99 -0
  10. package/dist/daemon-cli.d.ts +2 -0
  11. package/dist/daemon-cli.js +54 -0
  12. package/dist/daemon.d.ts +23 -0
  13. package/dist/daemon.js +1078 -0
  14. package/dist/embedding-store.d.ts +43 -0
  15. package/dist/embedding-store.js +169 -0
  16. package/dist/embeddings.d.ts +93 -0
  17. package/dist/embeddings.js +345 -0
  18. package/dist/entities.d.ts +61 -0
  19. package/dist/entities.js +191 -0
  20. package/dist/entity-extraction.d.ts +33 -0
  21. package/dist/entity-extraction.js +75 -0
  22. package/dist/eval-metrics.d.ts +27 -0
  23. package/dist/eval-metrics.js +50 -0
  24. package/dist/evaluation.d.ts +58 -0
  25. package/dist/evaluation.js +244 -0
  26. package/dist/global-extraction.d.ts +15 -0
  27. package/dist/global-extraction.js +61 -0
  28. package/dist/global-memory.d.ts +43 -0
  29. package/dist/global-memory.js +306 -0
  30. package/dist/global-promotion.d.ts +25 -0
  31. package/dist/global-promotion.js +29 -0
  32. package/dist/hyde.d.ts +31 -0
  33. package/dist/hyde.js +46 -0
  34. package/dist/index.d.ts +2 -0
  35. package/dist/index.js +246 -0
  36. package/dist/injection.d.ts +38 -0
  37. package/dist/injection.js +133 -0
  38. package/dist/logger.d.ts +17 -0
  39. package/dist/logger.js +63 -0
  40. package/dist/memory-mutations.d.ts +24 -0
  41. package/dist/memory-mutations.js +57 -0
  42. package/dist/memory-store.d.ts +194 -0
  43. package/dist/memory-store.js +1205 -0
  44. package/dist/monitor.d.ts +13 -0
  45. package/dist/monitor.js +977 -0
  46. package/dist/overview.d.ts +73 -0
  47. package/dist/overview.js +104 -0
  48. package/dist/processor.d.ts +90 -0
  49. package/dist/processor.js +450 -0
  50. package/dist/quality.d.ts +86 -0
  51. package/dist/quality.js +338 -0
  52. package/dist/recuration.d.ts +13 -0
  53. package/dist/recuration.js +65 -0
  54. package/dist/reranker.d.ts +34 -0
  55. package/dist/reranker.js +89 -0
  56. package/dist/retrieval.d.ts +106 -0
  57. package/dist/retrieval.js +392 -0
  58. package/dist/session-index.d.ts +34 -0
  59. package/dist/session-index.js +87 -0
  60. package/dist/temporal.d.ts +20 -0
  61. package/dist/temporal.js +62 -0
  62. package/dist/token-ab-monitor.d.ts +1 -0
  63. package/dist/token-ab-monitor.js +7 -0
  64. package/dist/tools.d.ts +232 -0
  65. package/dist/tools.js +546 -0
  66. package/dist/types.d.ts +169 -0
  67. package/dist/types.js +1 -0
  68. package/docs/assets/neural-universe.png +0 -0
  69. package/package.json +57 -0
  70. package/scripts/claude-peon-hook.mjs +522 -0
  71. package/scripts/codex-peon-hook.mjs +4 -0
  72. package/scripts/eval-retrieval-labeled.mjs +135 -0
  73. package/scripts/eval-retrieval.mjs +96 -0
  74. package/scripts/evaluate-peon.mjs +47 -0
  75. package/scripts/install-peon-stl.mjs +82 -0
  76. package/scripts/install-peon.mjs +318 -0
  77. package/scripts/lib/eval-ledger.mjs +104 -0
  78. package/scripts/lib/stl-classify.mjs +44 -0
  79. package/scripts/longmemeval-eval.mjs +144 -0
  80. package/scripts/peon-report.mjs +155 -0
  81. package/scripts/peon-stl.mjs +506 -0
  82. package/scripts/token-ab-monitor.html +235 -0
@@ -0,0 +1,338 @@
1
+ export function deduplicateMemoryRecords(records) {
2
+ const entries = [];
3
+ const keyToIndex = new Map();
4
+ const duplicates = [];
5
+ for (const record of records) {
6
+ const key = memoryKey(record);
7
+ const existingIndex = keyToIndex.get(key);
8
+ if (existingIndex === undefined) {
9
+ keyToIndex.set(key, entries.length);
10
+ entries.push({ key, record: cloneRecord(record) });
11
+ continue;
12
+ }
13
+ const current = entries[existingIndex].record;
14
+ const keepIncoming = memoryStrength(record) > memoryStrength(current);
15
+ const kept = keepIncoming ? mergeDuplicate(record, current) : mergeDuplicate(current, record);
16
+ entries[existingIndex] = { key, record: kept };
17
+ duplicates.push({
18
+ duplicateId: keepIncoming ? current.id : record.id,
19
+ keptId: kept.id,
20
+ key
21
+ });
22
+ }
23
+ return {
24
+ records: entries.map((entry) => entry.record),
25
+ duplicates
26
+ };
27
+ }
28
+ export function detectMemoryConflicts(records) {
29
+ const conflicts = [];
30
+ for (let leftIndex = 0; leftIndex < records.length; leftIndex += 1) {
31
+ for (let rightIndex = leftIndex + 1; rightIndex < records.length; rightIndex += 1) {
32
+ const left = records[leftIndex];
33
+ const right = records[rightIndex];
34
+ const entity = sharedEntity(left, right);
35
+ if (!entity)
36
+ continue;
37
+ const reason = opposingLanguageReason(left.content, right.content);
38
+ if (!reason)
39
+ continue;
40
+ conflicts.push({
41
+ entity,
42
+ leftId: left.id,
43
+ rightId: right.id,
44
+ reason
45
+ });
46
+ }
47
+ }
48
+ return conflicts;
49
+ }
50
+ export function markStaleMemoryRecords(records, options = {}) {
51
+ const now = options.now ?? new Date();
52
+ const staleAfterDays = options.staleAfterDays ?? 120;
53
+ const staleIds = [];
54
+ return {
55
+ records: records.map((record) => {
56
+ if (record.status !== "active")
57
+ return cloneRecord(record);
58
+ if (ageInDays(record.updatedAt, now) <= staleAfterDays)
59
+ return cloneRecord(record);
60
+ staleIds.push(record.id);
61
+ return {
62
+ ...cloneRecord(record),
63
+ status: "stale"
64
+ };
65
+ }),
66
+ staleIds
67
+ };
68
+ }
69
+ export function promoteMemoryRecords(records, options = {}) {
70
+ const repeatThreshold = options.repeatThreshold ?? 2;
71
+ const repeatBoost = options.repeatBoost ?? 0.2;
72
+ const importantBoost = options.importantBoost ?? 0.15;
73
+ const repeatedCounts = countNormalized(options.repeatedContent ?? []);
74
+ const importantTerms = (options.importantTerms ?? []).map(normalizeMemory).filter(Boolean);
75
+ const promoted = [];
76
+ const promotedRecords = records.map((record) => {
77
+ const normalized = normalizeMemory(record.content);
78
+ const isRepeated = (repeatedCounts.get(normalized) ?? 0) >= repeatThreshold;
79
+ const isImportant = importantTerms.some((term) => normalized.includes(term));
80
+ const boost = (isRepeated ? repeatBoost : 0) + (isImportant ? importantBoost : 0);
81
+ if (boost <= 0)
82
+ return cloneRecord(record);
83
+ const importance = clamp(record.score.importance + boost);
84
+ if (importance === record.score.importance)
85
+ return cloneRecord(record);
86
+ promoted.push({
87
+ id: record.id,
88
+ reason: isRepeated ? "repeated" : "important",
89
+ importance
90
+ });
91
+ return {
92
+ ...cloneRecord(record),
93
+ score: {
94
+ ...record.score,
95
+ importance
96
+ }
97
+ };
98
+ });
99
+ return { records: promotedRecords, promoted };
100
+ }
101
+ export function applyMemoryQualityReport(records, report) {
102
+ return {
103
+ records: report.records.map(cloneRecord),
104
+ audit: summarizeMemoryQualityReport(report, records)
105
+ };
106
+ }
107
+ export function summarizeMemoryQualityReport(report, records) {
108
+ const removedIds = uniquePreservingOrder(report.duplicates.map((duplicate) => duplicate.duplicateId));
109
+ const updatedIds = [];
110
+ const retainedIds = [];
111
+ if (records) {
112
+ const finalById = new Map(report.records.map((record) => [record.id, record]));
113
+ for (const record of records) {
114
+ const finalRecord = finalById.get(record.id);
115
+ if (!finalRecord) {
116
+ if (!removedIds.includes(record.id))
117
+ removedIds.push(record.id);
118
+ continue;
119
+ }
120
+ if (sameJsonSafeRecord(record, finalRecord)) {
121
+ retainedIds.push(record.id);
122
+ }
123
+ else {
124
+ updatedIds.push(record.id);
125
+ }
126
+ }
127
+ }
128
+ else {
129
+ const changedIds = new Set();
130
+ for (const id of report.staleIds)
131
+ changedIds.add(id);
132
+ for (const id of report.promotedIds)
133
+ changedIds.add(id);
134
+ for (const conflict of report.conflicts) {
135
+ changedIds.add(conflict.leftId);
136
+ changedIds.add(conflict.rightId);
137
+ }
138
+ for (const record of report.records) {
139
+ if (changedIds.has(record.id)) {
140
+ updatedIds.push(record.id);
141
+ }
142
+ else {
143
+ retainedIds.push(record.id);
144
+ }
145
+ }
146
+ }
147
+ return {
148
+ inputCount: report.inputCount,
149
+ outputCount: report.outputCount,
150
+ removedDuplicateCount: report.duplicates.length,
151
+ conflictCount: report.conflicts.length,
152
+ staleCount: report.staleIds.length,
153
+ promotedCount: report.promotedIds.length,
154
+ changedCount: updatedIds.length,
155
+ unchangedCount: retainedIds.length,
156
+ removedIds,
157
+ updatedIds,
158
+ retainedIds
159
+ };
160
+ }
161
+ export function serializeMemoryQualityReport(report) {
162
+ return {
163
+ inputCount: safeInteger(report.inputCount),
164
+ outputCount: safeInteger(report.outputCount),
165
+ records: report.records.map(jsonSafeRecord),
166
+ duplicates: report.duplicates.map((duplicate) => ({
167
+ duplicateId: String(duplicate.duplicateId),
168
+ keptId: String(duplicate.keptId),
169
+ key: String(duplicate.key)
170
+ })),
171
+ conflicts: report.conflicts.map((conflict) => ({
172
+ entity: String(conflict.entity),
173
+ leftId: String(conflict.leftId),
174
+ rightId: String(conflict.rightId),
175
+ reason: String(conflict.reason)
176
+ })),
177
+ staleIds: report.staleIds.map(String),
178
+ promotedIds: report.promotedIds.map(String),
179
+ audit: summarizeMemoryQualityReport(report)
180
+ };
181
+ }
182
+ export function createQualityReport(records, options = {}) {
183
+ const deduplicated = deduplicateMemoryRecords(records);
184
+ const stale = markStaleMemoryRecords(deduplicated.records, options);
185
+ const promoted = promoteMemoryRecords(stale.records, options);
186
+ // Superseded records are a settled verdict (a belief was explicitly replaced);
187
+ // never re-flag them as "conflicted" — that would clobber the supersede link.
188
+ const conflicts = detectMemoryConflicts(promoted.records.filter((record) => record.status !== "superseded"));
189
+ const conflictedIds = new Set(conflicts.flatMap((conflict) => [conflict.leftId, conflict.rightId]));
190
+ const finalRecords = promoted.records.map((record) => conflictedIds.has(record.id)
191
+ ? {
192
+ ...cloneRecord(record),
193
+ status: "conflicted"
194
+ }
195
+ : cloneRecord(record));
196
+ return {
197
+ inputCount: records.length,
198
+ outputCount: finalRecords.length,
199
+ records: finalRecords,
200
+ duplicates: deduplicated.duplicates,
201
+ conflicts,
202
+ staleIds: stale.staleIds,
203
+ promotedIds: promoted.promoted.map((record) => record.id)
204
+ };
205
+ }
206
+ function memoryKey(record) {
207
+ return `${record.type}:${normalizeMemory(record.content)}`;
208
+ }
209
+ function mergeDuplicate(kept, duplicate) {
210
+ return {
211
+ ...cloneRecord(kept),
212
+ score: {
213
+ importance: clamp(Math.max(kept.score.importance, duplicate.score.importance)),
214
+ confidence: clamp(Math.max(kept.score.confidence, duplicate.score.confidence))
215
+ },
216
+ entities: uniqueSorted([...kept.entities, ...duplicate.entities])
217
+ };
218
+ }
219
+ function memoryStrength(record) {
220
+ return record.score.importance + record.score.confidence + epochMillis(record.updatedAt) / 1_000_000_000_000_000;
221
+ }
222
+ function sharedEntity(left, right) {
223
+ const rightEntities = new Map(right.entities.map((entity) => [normalizeEntity(entity), entity]));
224
+ for (const leftEntity of left.entities) {
225
+ const match = rightEntities.get(normalizeEntity(leftEntity));
226
+ if (match)
227
+ return leftEntity.trim() || match;
228
+ }
229
+ return undefined;
230
+ }
231
+ function opposingLanguageReason(left, right) {
232
+ const leftText = normalizeMemory(left);
233
+ const rightText = normalizeMemory(right);
234
+ const pairs = [
235
+ ["enabled", "disabled", "opposing enabled/disabled language"],
236
+ ["enable", "disable", "opposing enable/disable language"],
237
+ ["allowed", "forbidden", "opposing allowed/forbidden language"],
238
+ ["allow", "deny", "opposing allow/deny language"],
239
+ ["required", "optional", "opposing required/optional language"],
240
+ ["true", "false", "opposing true/false language"],
241
+ ["yes", "no", "opposing yes/no language"],
242
+ ["use", "avoid", "opposing use/avoid language"]
243
+ ];
244
+ for (const [positive, negative, reason] of pairs) {
245
+ if (hasWord(leftText, positive) && hasWord(rightText, negative))
246
+ return reason;
247
+ if (hasWord(leftText, negative) && hasWord(rightText, positive))
248
+ return reason;
249
+ }
250
+ return undefined;
251
+ }
252
+ function ageInDays(value, now) {
253
+ const updatedAt = epochMillis(value);
254
+ if (!Number.isFinite(updatedAt))
255
+ return 0;
256
+ return (now.getTime() - updatedAt) / 86_400_000;
257
+ }
258
+ function epochMillis(value) {
259
+ const timestamp = Date.parse(value);
260
+ return Number.isFinite(timestamp) ? timestamp : 0;
261
+ }
262
+ function countNormalized(values) {
263
+ const counts = new Map();
264
+ for (const value of values) {
265
+ const normalized = normalizeMemory(value);
266
+ if (!normalized)
267
+ continue;
268
+ counts.set(normalized, (counts.get(normalized) ?? 0) + 1);
269
+ }
270
+ return counts;
271
+ }
272
+ function normalizeMemory(content) {
273
+ return content
274
+ .toLowerCase()
275
+ .replace(/[`"'.,;:!?()[\]{}]/g, "")
276
+ .replace(/\s+/g, " ")
277
+ .trim();
278
+ }
279
+ function normalizeEntity(entity) {
280
+ return normalizeMemory(entity);
281
+ }
282
+ function hasWord(content, word) {
283
+ return new RegExp(`(^|\\s)${escapeRegExp(word)}($|\\s)`).test(content);
284
+ }
285
+ function escapeRegExp(value) {
286
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
287
+ }
288
+ function uniqueSorted(values) {
289
+ return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean))).sort((left, right) => left.localeCompare(right));
290
+ }
291
+ function uniquePreservingOrder(values) {
292
+ const seen = new Set();
293
+ return values.filter((value) => {
294
+ if (seen.has(value))
295
+ return false;
296
+ seen.add(value);
297
+ return true;
298
+ });
299
+ }
300
+ function cloneRecord(record) {
301
+ return {
302
+ ...record,
303
+ score: { ...record.score },
304
+ source: { ...record.source },
305
+ entities: [...record.entities]
306
+ };
307
+ }
308
+ function clamp(value) {
309
+ return Math.max(0, Math.min(1, Number.isFinite(value) ? value : 0));
310
+ }
311
+ function safeInteger(value) {
312
+ return Number.isSafeInteger(value) ? value : 0;
313
+ }
314
+ function sameJsonSafeRecord(left, right) {
315
+ return JSON.stringify(jsonSafeRecord(left)) === JSON.stringify(jsonSafeRecord(right));
316
+ }
317
+ function jsonSafeRecord(record) {
318
+ return {
319
+ id: String(record.id),
320
+ type: record.type,
321
+ content: String(record.content),
322
+ normalized: String(record.normalized),
323
+ scope: record.scope,
324
+ status: record.status,
325
+ score: {
326
+ importance: clamp(record.score.importance),
327
+ confidence: clamp(record.score.confidence)
328
+ },
329
+ source: {
330
+ kind: record.source.kind,
331
+ ...(record.source.reason === undefined ? {} : { reason: String(record.source.reason) })
332
+ },
333
+ entities: record.entities.map(String),
334
+ createdAt: String(record.createdAt),
335
+ updatedAt: String(record.updatedAt),
336
+ ...(record.supersededBy === undefined ? {} : { supersededBy: String(record.supersededBy) })
337
+ };
338
+ }
@@ -0,0 +1,13 @@
1
+ import type { PeonConfig } from "./config.js";
2
+ import type { MemoryRecord } from "./types.js";
3
+ /**
4
+ * One-time cleanup: re-judge a project's EXISTING beliefs against the sharpened
5
+ * rules and return the ids to retire. Operates on already-distilled beliefs (not
6
+ * raw events) so it's cheap, preserves supersession history, and only removes the
7
+ * ephemeral/trivial/duplicate noise the old prompt let through. Removed beliefs are
8
+ * archived (recoverable), never deleted.
9
+ */
10
+ export type Recurator = (records: readonly MemoryRecord[]) => Promise<string[]>;
11
+ export declare function createRecurator(config: PeonConfig): Recurator | null;
12
+ /** Tolerant parse of the model's id list. */
13
+ export declare function parseIdArray(content: string): string[];
@@ -0,0 +1,65 @@
1
+ const BATCH_SIZE = 30;
2
+ const PER_BATCH_MAX_FRACTION = 0.4; // a batch that wants to drop more than this is misjudging — skip it
3
+ export function createRecurator(config) {
4
+ if (config.aiMode === "off" || !config.openRouterApiKey)
5
+ return null;
6
+ return async (records) => {
7
+ const active = records.filter((r) => r.status === "active" && !r.pinned);
8
+ if (active.length === 0)
9
+ return [];
10
+ // Judge in small batches — the model reasons reliably over ~30 beliefs but
11
+ // over-selects wildly when handed hundreds at once.
12
+ const drop = [];
13
+ for (let i = 0; i < active.length; i += BATCH_SIZE) {
14
+ const batch = active.slice(i, i + BATCH_SIZE);
15
+ const ids = await judgeBatch(config, batch).catch(() => []);
16
+ const valid = ids.filter((id) => batch.some((r) => r.id === id));
17
+ // Per-batch rogue guard: if a batch wants to drop most of itself, skip it entirely.
18
+ if (valid.length > Math.max(3, Math.floor(batch.length * PER_BATCH_MAX_FRACTION)))
19
+ continue;
20
+ drop.push(...valid);
21
+ }
22
+ return drop;
23
+ };
24
+ }
25
+ async function judgeBatch(config, batch) {
26
+ const list = batch.map((r) => `${r.id} | ${r.type} | ${r.content}`).join("\n");
27
+ const system = "You are CONSERVATIVELY trimming a project's memory. Each line is 'id | type | content'. " +
28
+ "Default action is KEEP. Remove a belief ONLY if it is unmistakably one of: " +
29
+ "(a) an ephemeral one-time ACTION with no lasting meaning — 'cloned the repo', 'installed deps', 'created a directory', " +
30
+ "'downloaded weights', 'submitted/monitored a job', 'confirmed the job is pending', 'added a warning filter'; OR " +
31
+ "(b) an EXACT duplicate of another belief in the list (keep one, remove the literal repeats). " +
32
+ "Do NOT remove anything for being merely 'less important', verbose, or arguable — only clear noise and exact dupes. " +
33
+ "KEEP every decision, result/metric, preference, file, and open question unless it is pure setup-action noise. " +
34
+ "You should typically remove only a small fraction; removing most beliefs is WRONG. When in any doubt, KEEP. " +
35
+ "Output ONLY a JSON array of the ids to remove — no fences, no prose. If none, return [].";
36
+ const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
37
+ method: "POST",
38
+ headers: { Authorization: `Bearer ${config.openRouterApiKey}`, "Content-Type": "application/json" },
39
+ body: JSON.stringify({
40
+ model: config.processingModel,
41
+ messages: [
42
+ { role: "system", content: system },
43
+ { role: "user", content: `Beliefs:\n${list}\n\nIds to remove (JSON array):` }
44
+ ],
45
+ temperature: 0.1
46
+ })
47
+ });
48
+ if (!response.ok)
49
+ throw new Error(`recuration failed with ${response.status}`);
50
+ const json = (await response.json());
51
+ return parseIdArray(json.choices?.[0]?.message?.content ?? "");
52
+ }
53
+ /** Tolerant parse of the model's id list. */
54
+ export function parseIdArray(content) {
55
+ const text = content.trim();
56
+ const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
57
+ const body = fenced ? fenced[1] : text.slice(text.indexOf("["), text.lastIndexOf("]") + 1);
58
+ try {
59
+ const parsed = JSON.parse(body || text);
60
+ return Array.isArray(parsed) ? parsed.filter((x) => typeof x === "string") : [];
61
+ }
62
+ catch {
63
+ return [];
64
+ }
65
+ }
@@ -0,0 +1,34 @@
1
+ import type { PeonConfig } from "./config.js";
2
+ import type { RankedMemoryRecord } from "./retrieval.js";
3
+ /**
4
+ * Stage two of two-stage retrieval. The lexical+semantic ranker (RRF) is a high-RECALL
5
+ * first pass: cheap, deterministic, surfaces everything plausibly relevant. This reranker
6
+ * is the high-PRECISION second pass: a small, fast LLM (flash-lite) reads the actual query
7
+ * and the candidate snippets and reorders the top-K by true relevance — catching meaning
8
+ * that token overlap and cosine miss (negation, intent, the right sense of an ambiguous term).
9
+ *
10
+ * It is strictly optional and degrades gracefully: with no API key, AI disabled, an empty
11
+ * query, too few candidates, or any LLM/parse failure, it returns the input order unchanged.
12
+ * Only the top-K head is reranked; the tail is left in its original order and appended, so a
13
+ * truncated or partial model response can never drop or duplicate a candidate.
14
+ */
15
+ export type FetchLike = (url: string, init: RequestInit) => Promise<{
16
+ ok: boolean;
17
+ status: number;
18
+ text(): Promise<string>;
19
+ json(): Promise<unknown>;
20
+ }>;
21
+ export interface RerankOptions {
22
+ config: PeonConfig;
23
+ /** How many of the top candidates to rerank. Default 20. The rest keep their order. */
24
+ topK?: number;
25
+ /** Override the rerank model. Defaults to the (cheap) processing model. */
26
+ model?: string;
27
+ /** Injectable fetch — defaults to global fetch. Lets tests run without a network. */
28
+ fetchImpl?: FetchLike;
29
+ /** Per-snippet character cap fed to the model (keeps the prompt — and cost — small). Default 240. */
30
+ snippetChars?: number;
31
+ }
32
+ export declare function rerankRecords(query: string | undefined, records: RankedMemoryRecord[], options: RerankOptions): Promise<RankedMemoryRecord[]>;
33
+ /** Extract the leading JSON array of positive integers from a model response, tolerant of stray prose/fences. */
34
+ export declare function parseOrder(content: string, max: number): number[];
@@ -0,0 +1,89 @@
1
+ const DEFAULT_TOP_K = 20;
2
+ const DEFAULT_SNIPPET_CHARS = 240;
3
+ export async function rerankRecords(query, records, options) {
4
+ const { config } = options;
5
+ const q = (query ?? "").trim();
6
+ // Cheap exits — never pay an LLM call when it cannot help.
7
+ if (!q || records.length < 2)
8
+ return records;
9
+ if (config.aiMode === "off" || !config.openRouterApiKey)
10
+ return records;
11
+ const topK = Math.max(2, Math.trunc(options.topK ?? DEFAULT_TOP_K));
12
+ const snippetChars = Math.max(40, Math.trunc(options.snippetChars ?? DEFAULT_SNIPPET_CHARS));
13
+ const head = records.slice(0, topK);
14
+ const tail = records.slice(topK);
15
+ const doFetch = options.fetchImpl ?? globalThis.fetch;
16
+ if (!doFetch)
17
+ return records;
18
+ const numbered = head
19
+ .map((item, i) => `${i + 1}. [${item.record.type}] ${truncate(item.record.content, snippetChars)}`)
20
+ .join("\n");
21
+ const system = "You are a precision reranker for a memory system. Given a user query and a numbered list " +
22
+ "of candidate memory snippets, decide which snippets best help answer the query. " +
23
+ "Return ONLY a JSON array of the candidate numbers, ordered from most to least relevant. " +
24
+ "Include every number exactly once. No prose, no code fences.";
25
+ const user = `Query: ${q}\n\nCandidates:\n${numbered}\n\nJSON array of numbers (most relevant first):`;
26
+ try {
27
+ const response = await doFetch("https://openrouter.ai/api/v1/chat/completions", {
28
+ method: "POST",
29
+ headers: {
30
+ Authorization: `Bearer ${config.openRouterApiKey}`,
31
+ "Content-Type": "application/json"
32
+ },
33
+ body: JSON.stringify({
34
+ model: options.model ?? config.processingModel,
35
+ messages: [
36
+ { role: "system", content: system },
37
+ { role: "user", content: user }
38
+ ],
39
+ temperature: 0
40
+ })
41
+ });
42
+ if (!response.ok)
43
+ return records;
44
+ const json = (await response.json());
45
+ const content = json.choices?.[0]?.message?.content ?? "";
46
+ const order = parseOrder(content, head.length);
47
+ if (order.length === 0)
48
+ return records;
49
+ // Apply the model's order to the head, then append any head items it omitted (in their
50
+ // original order), then the untouched tail. Guarantees a permutation of the input.
51
+ const seen = new Set();
52
+ const reorderedHead = [];
53
+ for (const n of order) {
54
+ const idx = n - 1;
55
+ if (idx >= 0 && idx < head.length && !seen.has(idx)) {
56
+ seen.add(idx);
57
+ reorderedHead.push(head[idx]);
58
+ }
59
+ }
60
+ for (let i = 0; i < head.length; i++)
61
+ if (!seen.has(i))
62
+ reorderedHead.push(head[i]);
63
+ return [...reorderedHead, ...tail];
64
+ }
65
+ catch {
66
+ return records; // any failure → first-pass order, never worse than before
67
+ }
68
+ }
69
+ /** Extract the leading JSON array of positive integers from a model response, tolerant of stray prose/fences. */
70
+ export function parseOrder(content, max) {
71
+ const match = content.match(/\[[\s\S]*?\]/);
72
+ if (!match)
73
+ return [];
74
+ try {
75
+ const parsed = JSON.parse(match[0]);
76
+ if (!Array.isArray(parsed))
77
+ return [];
78
+ return parsed
79
+ .map((n) => (typeof n === "number" ? Math.trunc(n) : Number.parseInt(String(n), 10)))
80
+ .filter((n) => Number.isInteger(n) && n >= 1 && n <= max);
81
+ }
82
+ catch {
83
+ return [];
84
+ }
85
+ }
86
+ function truncate(text, max) {
87
+ const flat = text.replace(/\s+/g, " ").trim();
88
+ return flat.length <= max ? flat : `${flat.slice(0, max - 1)}…`;
89
+ }
@@ -0,0 +1,106 @@
1
+ import { type EmbeddingVector } from "./embeddings.js";
2
+ import type { MemoryRecord, MemoryType } from "./types.js";
3
+ export type RetrievalReasonKind = "query_term" | "entity" | "file" | "type" | "quality" | "recency" | "status" | "semantic";
4
+ export interface RetrievalReason {
5
+ kind: RetrievalReasonKind;
6
+ label: string;
7
+ score: number;
8
+ }
9
+ export interface RankedMemoryRecord {
10
+ record: MemoryRecord;
11
+ score: number;
12
+ reasons: RetrievalReason[];
13
+ explanation: string;
14
+ }
15
+ export interface SemanticRetrievalInput {
16
+ /** Embedding of the query string. */
17
+ queryVector: EmbeddingVector;
18
+ /** Record id → its stored embedding vector. */
19
+ vectorById: Map<string, EmbeddingVector>;
20
+ /** How strongly cosine similarity counts toward the final score. Default 6. */
21
+ weight?: number;
22
+ /** Cosine similarities below this contribute nothing (filters noise). Default 0.15. */
23
+ minSimilarity?: number;
24
+ }
25
+ export interface RetrievalOptions {
26
+ now?: Date | string | number;
27
+ limit?: number;
28
+ typeWeights?: Partial<Record<MemoryType, number>>;
29
+ /** When provided (and a query is given), blends semantic similarity into ranking. */
30
+ semantic?: SemanticRetrievalInput;
31
+ /** Per-record entity-graph activation (id → score). Fused as a signal AND admits an
32
+ * associatively-activated belief through the relevance gate, so a strong association can
33
+ * enter the top-K instead of being appended after the direct hits. See computeGraphActivation. */
34
+ graphActivation?: Map<string, number>;
35
+ }
36
+ export interface ContextSelectionOptions {
37
+ maxChars: number;
38
+ recordFormatter?: (item: RankedMemoryRecord) => string;
39
+ }
40
+ export interface ContextSelection {
41
+ records: RankedMemoryRecord[];
42
+ omitted: RankedMemoryRecord[];
43
+ totalChars: number;
44
+ maxChars: number;
45
+ }
46
+ /**
47
+ * Hybrid retrieval by Reciprocal Rank Fusion. Each signal (lexical, semantic,
48
+ * quality, recency, reinforcement strength) ranks the relevance-gated candidates
49
+ * independently; we fuse by Σ 1/(k+rank). RRF is weight-free — it removes the
50
+ * fragile hand-tuned score constants by comparing RANKS, which are commensurable
51
+ * across signals where raw scores are not. Pinned beliefs are boosted; archived/
52
+ * superseded are excluded by default (they are the long-term tier, not working memory).
53
+ */
54
+ export declare function rankMemoryRecords(records: MemoryRecord[], query: string | undefined, options?: RetrievalOptions): RankedMemoryRecord[];
55
+ export declare function selectMemoryRecordsForContext(rankedRecords: RankedMemoryRecord[], options: ContextSelectionOptions): ContextSelection;
56
+ /** Default trade-off: 0.7 weight on relevance, 0.3 on novelty. Tuned for coverage without losing the top hit. */
57
+ export declare const DEFAULT_MMR_LAMBDA = 0.7;
58
+ /**
59
+ * Re-order ranked records by Maximal Marginal Relevance so the injected block has
60
+ * COVERAGE instead of five paraphrases of the same belief. Each pick maximizes
61
+ * λ·relevance − (1−λ)·maxSimilarity(candidate, alreadyPicked)
62
+ * Relevance is the record's existing fused score (max-normalized to [0,1] within the
63
+ * set); similarity is lexical Jaccard over content+entity tokens — cheap, deterministic,
64
+ * and embedding-free so it works in pure local-first mode. The single most relevant
65
+ * record is always selected first, so the top hit is never displaced by diversification.
66
+ */
67
+ export declare function diversifyByMMR(records: RankedMemoryRecord[], lambda?: number): RankedMemoryRecord[];
68
+ export interface GraphExpandOptions {
69
+ /** How many of the top ranked records seed the activation. Default 8. */
70
+ seedDepth?: number;
71
+ /** Max neighbours to pull in. Default 6. */
72
+ maxNeighbors?: number;
73
+ /** Global decay on the 1-hop spread (keeps neighbours supplementary). Default 0.5. */
74
+ damping?: number;
75
+ /** Weight for code-namespace entities vs domain (file/symbol co-occurrence is weaker signal). Default 0.4. */
76
+ codeWeight?: number;
77
+ /** Skip entities mentioned by more than this many beliefs — super-hub hairball guard. Default 40. */
78
+ hubDegreeCap?: number;
79
+ }
80
+ /**
81
+ * Entity-graph spreading activation (the associative-recall layer). Lexical/semantic ranking
82
+ * finds beliefs that match the QUERY; this finds beliefs in the ANSWER's neighbourhood by
83
+ * spreading activation from the top hits through shared entities, with three brain-like rules
84
+ * the old flat 1-hop expander lacked:
85
+ * - DISTANCE DECAY — a global damping (λ) keeps neighbours below direct matches.
86
+ * - MULTI-SOURCE SUMMATION — a belief lit through several shared entities (or several seeds)
87
+ * accumulates activation, so it outranks one lit through a single weak link.
88
+ * - HUB DAMPING — rare entities transmit more activation (1/log₂(2+degree)); super-hubs
89
+ * (e.g. a file mentioned by 80 beliefs) are skipped so the graph isn't a hairball.
90
+ * Domain entities (people/papers/concepts) spread more than code entities. Pure + deterministic.
91
+ * Neighbours come back with small scores in their own band and are meant to be appended AFTER
92
+ * the direct results, never displacing them.
93
+ */
94
+ /**
95
+ * Raw spreading activation: id → accumulated activation for beliefs in the seeds' entity
96
+ * neighbourhood (excluding the seeds themselves). The associative substrate, shared by the
97
+ * fused-ranking path (passed as RetrievalOptions.graphActivation) and the legacy append path
98
+ * (expandByEntityGraph). Excludes seed ids so direct hits aren't double-counted.
99
+ */
100
+ export declare function computeGraphActivation(seeds: RankedMemoryRecord[], pool: MemoryRecord[], options?: GraphExpandOptions): Map<string, number>;
101
+ /**
102
+ * Entity-graph spreading activation, formatted as standalone neighbour records (legacy append
103
+ * path + the unit tests). The fused-ranking path uses computeGraphActivation directly via
104
+ * rankMemoryRecords' graphActivation option, which lets associations compete inside the top-K.
105
+ */
106
+ export declare function expandByEntityGraph(seeds: RankedMemoryRecord[], pool: MemoryRecord[], options?: GraphExpandOptions): RankedMemoryRecord[];