recallnest 2.5.4 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +34 -9
- package/README_CN.md +37 -12
- package/package.json +4 -4
- package/src/__tests__/access-tracker-novelty.test.ts +66 -0
- package/src/__tests__/api-server-security.test.ts +102 -0
- package/src/__tests__/belief-history.test.ts +220 -0
- package/src/__tests__/capture-engine.test.ts +93 -3
- package/src/__tests__/category-min-scores.test.ts +83 -0
- package/src/__tests__/cluster-consolidation.test.ts +167 -4
- package/src/__tests__/conflict-store.test.ts +46 -2
- package/src/__tests__/consolidation-engine.test.ts +97 -0
- package/src/__tests__/contracts.test.ts +46 -0
- package/src/__tests__/decay-exemption.test.ts +45 -0
- package/src/__tests__/dream-pipeline.test.ts +464 -5
- package/src/__tests__/freshness.test.ts +186 -0
- package/src/__tests__/hotness-blend.test.ts +207 -0
- package/src/__tests__/llm-client-pivot.test.ts +628 -0
- package/src/__tests__/maintenance-locks.test.ts +15 -0
- package/src/__tests__/memory-boundaries.test.ts +32 -0
- package/src/__tests__/noise-prototype-bank.test.ts +280 -0
- package/src/__tests__/pii-redact.test.ts +19 -0
- package/src/__tests__/pivot-apply.test.ts +373 -0
- package/src/__tests__/pivot-distill-supervisor.test.ts +457 -0
- package/src/__tests__/pivot-distill.test.ts +1178 -0
- package/src/__tests__/preference-matcher.test.ts +54 -2
- package/src/__tests__/pull-from-macbook-sync-only.test.ts +679 -0
- package/src/__tests__/recall-gate.test.ts +266 -0
- package/src/__tests__/resolve-db-path.test.ts +130 -0
- package/src/__tests__/resolve-tier.test.ts +64 -0
- package/src/__tests__/retrieval-eval.test.ts +163 -0
- package/src/__tests__/retriever-lazy-init-dispatch.test.ts +123 -0
- package/src/__tests__/skill-promotion.test.ts +53 -0
- package/src/__tests__/stable-context-length-contract.test.ts +62 -0
- package/src/__tests__/store-read-consistency.test.ts +122 -0
- package/src/__tests__/store-schema-migration.test.ts +87 -0
- package/src/__tests__/store-upsert.test.ts +17 -0
- package/src/__tests__/term-registry-terminal-roster.test.ts +127 -0
- package/src/__tests__/ui-server-security.test.ts +93 -0
- package/src/access-tracker.ts +53 -3
- package/src/adaptive-retrieval.ts +11 -0
- package/src/api-server.ts +9 -2
- package/src/auto-recall.ts +61 -1
- package/src/belief-history.ts +164 -0
- package/src/capture-engine.ts +88 -11
- package/src/cli.ts +109 -14
- package/src/conflict-engine.ts +24 -3
- package/src/consolidation-engine.ts +146 -12
- package/src/context-composer-stable-selection.ts +7 -1
- package/src/data-checkup.ts +1 -1
- package/src/decay-engine.ts +59 -7
- package/src/doctor.ts +4 -2
- package/src/dream-pipeline.ts +359 -11
- package/src/env-config.ts +41 -0
- package/src/eval.ts +154 -11
- package/src/freshness.ts +192 -0
- package/src/http-request-policy.ts +148 -0
- package/src/ingest.ts +277 -2
- package/src/kg-extractor.ts +7 -1
- package/src/llm-client.ts +106 -11
- package/src/mcp-server.ts +2 -1
- package/src/mcp-tools-advanced.ts +13 -0
- package/src/mcp-tools-core.ts +5 -1
- package/src/memory-boundaries.ts +23 -2
- package/src/memory-health-rebalance.ts +4 -0
- package/src/memory-output.ts +16 -2
- package/src/memory-store-port.ts +3 -0
- package/src/noise-prototype-bank.ts +408 -0
- package/src/pii-detector.ts +52 -2
- package/src/pivot-apply.ts +399 -0
- package/src/preference-matcher.ts +37 -3
- package/src/recall-gate.ts +305 -0
- package/src/retrieval-profiles.ts +46 -4
- package/src/retriever.ts +95 -18
- package/src/retroactive-boost.ts +6 -2
- package/src/runtime-config.ts +70 -5
- package/src/skill-promotion.ts +58 -2
- package/src/store.ts +54 -19
- package/src/term-registry.ts +35 -0
- package/src/ui-server.ts +7 -0
- package/src/version.ts +19 -0
package/src/ingest.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { isProcessed, markProcessed } from "./tracker.js";
|
|
|
24
24
|
import * as envConfig from "./env-config.js";
|
|
25
25
|
import type { LLMClient, SmartExtraction } from "./llm-client.js";
|
|
26
26
|
import { resolveIngestBoundary } from "./memory-boundaries.js";
|
|
27
|
+
import { isActiveMemory } from "./memory-evolution.js";
|
|
27
28
|
import { isNoise } from "./noise-filter.js";
|
|
28
29
|
import {
|
|
29
30
|
inferReplyStylePreferenceSlot,
|
|
@@ -289,8 +290,12 @@ export async function dedupCheck(
|
|
|
289
290
|
llm?: LLMClient | null,
|
|
290
291
|
): Promise<DedupCheckResult> {
|
|
291
292
|
try {
|
|
292
|
-
// Stage 1: vector similarity check
|
|
293
|
-
|
|
293
|
+
// Stage 1: vector similarity check. store.vectorSearch() has no status filter, so
|
|
294
|
+
// superseded belief-history rows surface here — dedupe against one and incoming
|
|
295
|
+
// content gets dropped as a duplicate of a belief that is no longer held (returning
|
|
296
|
+
// to an earlier position would be silently discarded).
|
|
297
|
+
const results = (await store.vectorSearch(vector, DEDUP_CANDIDATE_LIMIT, DEDUP_SOFT_THRESHOLD))
|
|
298
|
+
.filter((result) => isActiveMemory(result.entry.metadata));
|
|
294
299
|
if (results.length === 0) {
|
|
295
300
|
return { action: "store", reason: "unique" }; // Clearly unique
|
|
296
301
|
}
|
|
@@ -1190,6 +1195,276 @@ export async function ingestCodexSessions(
|
|
|
1190
1195
|
return result;
|
|
1191
1196
|
}
|
|
1192
1197
|
|
|
1198
|
+
// ============================================================================
|
|
1199
|
+
// Kimi Code Session Parser
|
|
1200
|
+
// ============================================================================
|
|
1201
|
+
// Kimi Code 的 wire.jsonl 是事件流(非 role/content 行),对话内容只在三类事件里:
|
|
1202
|
+
// turn.prompt (origin.kind="user") → 用户输入,.input 是 text part 数组
|
|
1203
|
+
// context.append_message → 消息;用户消息只收 origin.kind="user"
|
|
1204
|
+
// (injection/skill_activation 等是系统注入,跳过)
|
|
1205
|
+
// context.append_loop_event content.part → 助手文本(part.type==="text";think 跳过)
|
|
1206
|
+
// 其余类型(llm.tools_snapshot / mcp.tools_discovered / config.update / llm.request /
|
|
1207
|
+
// usage.record / permission.* 等,约占 86% 体积)是运行事件,一律跳过。
|
|
1208
|
+
function parseKimiSession(filePath: string): ConversationTurn[] {
|
|
1209
|
+
const turns: ConversationTurn[] = [];
|
|
1210
|
+
const content = readFileSync(filePath, "utf-8");
|
|
1211
|
+
const lines = content.split("\n").filter((l) => l.trim());
|
|
1212
|
+
|
|
1213
|
+
// sessionId 从路径取:.../sessions/<workDirKey>/<sessionId>/agents/main/wire.jsonl
|
|
1214
|
+
// 去掉 "session_" 前缀,否则 scope 统一变成 kimi:session_
|
|
1215
|
+
const pathParts = filePath.split("/");
|
|
1216
|
+
const rawId = pathParts.length >= 4 ? pathParts[pathParts.length - 4] : basename(filePath, ".jsonl");
|
|
1217
|
+
const sessionId = rawId.replace(/^session_/, "");
|
|
1218
|
+
|
|
1219
|
+
const pushTurn = (role: "user" | "assistant", text: string, timestamp: string) => {
|
|
1220
|
+
const trimmed = text.trim();
|
|
1221
|
+
// 中文 8 字符已是完整句;用户阈值与 codex/cc 对齐
|
|
1222
|
+
if (trimmed.length <= (role === "user" ? 10 : 8)) return;
|
|
1223
|
+
// turn.prompt 与随后的 context.append_message 重复记录同一用户输入(同 codex 相邻去重)
|
|
1224
|
+
const lastTurn = turns[turns.length - 1];
|
|
1225
|
+
if (lastTurn && lastTurn.role === role && lastTurn.text === trimmed) return;
|
|
1226
|
+
turns.push({ role, text: trimmed, timestamp, sessionId });
|
|
1227
|
+
};
|
|
1228
|
+
|
|
1229
|
+
for (const line of lines) {
|
|
1230
|
+
try {
|
|
1231
|
+
const obj = JSON.parse(line);
|
|
1232
|
+
const timestamp = typeof obj.time === "number" ? new Date(obj.time).toISOString() : "";
|
|
1233
|
+
|
|
1234
|
+
if (obj.type === "turn.prompt") {
|
|
1235
|
+
if (obj.origin?.kind !== "user") continue;
|
|
1236
|
+
const parts = Array.isArray(obj.input) ? obj.input : [];
|
|
1237
|
+
const text = parts
|
|
1238
|
+
.filter((c: any) => c?.type === "text")
|
|
1239
|
+
.map((c: any) => c.text || "")
|
|
1240
|
+
.join("\n");
|
|
1241
|
+
if (text) pushTurn("user", text, timestamp);
|
|
1242
|
+
continue;
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
if (obj.type === "context.append_message") {
|
|
1246
|
+
const msg = obj.message;
|
|
1247
|
+
if (!msg || (msg.role !== "user" && msg.role !== "assistant")) continue;
|
|
1248
|
+
// 系统注入(system-reminder、skill 激活、后台任务通知)不是用户对话内容
|
|
1249
|
+
if (msg.role === "user" && msg.origin?.kind !== "user") continue;
|
|
1250
|
+
let text = "";
|
|
1251
|
+
if (typeof msg.content === "string") {
|
|
1252
|
+
text = msg.content;
|
|
1253
|
+
} else if (Array.isArray(msg.content)) {
|
|
1254
|
+
text = msg.content
|
|
1255
|
+
.filter((c: any) => c.type === "text")
|
|
1256
|
+
.map((c: any) => c.text || "")
|
|
1257
|
+
.join("\n");
|
|
1258
|
+
}
|
|
1259
|
+
if (text) pushTurn(msg.role, text, timestamp);
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
if (obj.type === "context.append_loop_event") {
|
|
1264
|
+
const part = obj.event?.type === "content.part" ? obj.event.part : null;
|
|
1265
|
+
if (part?.type === "text" && typeof part.text === "string") {
|
|
1266
|
+
pushTurn("assistant", part.text, timestamp);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
} catch {
|
|
1270
|
+
// Skip malformed lines
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
return turns;
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
export async function ingestKimiSessions(
|
|
1278
|
+
store: MemoryStore,
|
|
1279
|
+
embedder: Embedder,
|
|
1280
|
+
options: { limit?: number; verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null; recentHours?: number } = {},
|
|
1281
|
+
): Promise<IngestResult> {
|
|
1282
|
+
const result: IngestResult = {
|
|
1283
|
+
source: "kimi",
|
|
1284
|
+
filesProcessed: 0,
|
|
1285
|
+
chunksIngested: 0,
|
|
1286
|
+
chunksSkipped: 0,
|
|
1287
|
+
chunksDeduped: 0,
|
|
1288
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
1289
|
+
errors: [],
|
|
1290
|
+
};
|
|
1291
|
+
|
|
1292
|
+
const baseDir = expandHome("~/.kimi-code/sessions");
|
|
1293
|
+
const indexPath = expandHome("~/.kimi-code/session_index.jsonl");
|
|
1294
|
+
|
|
1295
|
+
// 文件发现:优先 session_index.jsonl(精确),再 walk sessions 目录兜底(索引缺漏时)。
|
|
1296
|
+
// 只收 agents/main/wire.jsonl,子 agent 的 agents/agent-N/wire.jsonl 与主对话重复,不采。
|
|
1297
|
+
const seen = new Set<string>();
|
|
1298
|
+
const allFiles: string[] = [];
|
|
1299
|
+
const addFile = (p: string) => {
|
|
1300
|
+
if (!seen.has(p) && existsSync(p)) {
|
|
1301
|
+
seen.add(p);
|
|
1302
|
+
allFiles.push(p);
|
|
1303
|
+
}
|
|
1304
|
+
};
|
|
1305
|
+
|
|
1306
|
+
if (existsSync(indexPath)) {
|
|
1307
|
+
try {
|
|
1308
|
+
for (const line of readFileSync(indexPath, "utf-8").split("\n")) {
|
|
1309
|
+
if (!line.trim()) continue;
|
|
1310
|
+
try {
|
|
1311
|
+
const entry = JSON.parse(line);
|
|
1312
|
+
if (typeof entry.sessionDir === "string" && entry.sessionDir) {
|
|
1313
|
+
addFile(join(entry.sessionDir, "agents", "main", "wire.jsonl"));
|
|
1314
|
+
}
|
|
1315
|
+
} catch {
|
|
1316
|
+
// Skip malformed index lines
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
} catch {
|
|
1320
|
+
// 索引读不出来就走下面的目录扫描
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
if (existsSync(baseDir)) {
|
|
1325
|
+
const mainWireSuffix = join("agents", "main", "wire.jsonl");
|
|
1326
|
+
const walk = (dir: string) => {
|
|
1327
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1328
|
+
const full = join(dir, entry.name);
|
|
1329
|
+
if (entry.isDirectory()) {
|
|
1330
|
+
walk(full);
|
|
1331
|
+
} else if (full.endsWith(mainWireSuffix)) {
|
|
1332
|
+
addFile(full);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
};
|
|
1336
|
+
walk(baseDir);
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
if (allFiles.length === 0) {
|
|
1340
|
+
result.errors.push(`Kimi Code sessions not found: ${baseDir} (index: ${indexPath})`);
|
|
1341
|
+
return result;
|
|
1342
|
+
}
|
|
1343
|
+
allFiles.sort();
|
|
1344
|
+
|
|
1345
|
+
const recentMode = options.recentHours !== undefined;
|
|
1346
|
+
const cutoffMs = recentMode ? Date.now() - options.recentHours! * 3600_000 : 0;
|
|
1347
|
+
|
|
1348
|
+
// --recent: only keep files modified within N hours
|
|
1349
|
+
const filteredFiles = recentMode
|
|
1350
|
+
? allFiles.filter((f) => { try { return statSync(f).mtimeMs >= cutoffMs; } catch { return false; } })
|
|
1351
|
+
: allFiles;
|
|
1352
|
+
|
|
1353
|
+
const filesToProcess = options.limit ? filteredFiles.slice(0, options.limit) : filteredFiles;
|
|
1354
|
+
const total = filesToProcess.length;
|
|
1355
|
+
|
|
1356
|
+
for (let fi = 0; fi < filesToProcess.length; fi++) {
|
|
1357
|
+
const filePath = filesToProcess[fi];
|
|
1358
|
+
|
|
1359
|
+
try {
|
|
1360
|
+
const stat = statSync(filePath);
|
|
1361
|
+
if (stat.size < 200) {
|
|
1362
|
+
result.chunksSkipped++;
|
|
1363
|
+
continue;
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
// In --recent mode, skip ingested-files check — files may have new content appended
|
|
1367
|
+
if (!recentMode && isProcessed(filePath, stat.size, stat.mtimeMs)) {
|
|
1368
|
+
result.chunksSkipped++;
|
|
1369
|
+
continue;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
const turns = parseKimiSession(filePath);
|
|
1373
|
+
if (turns.length === 0) {
|
|
1374
|
+
result.chunksSkipped++;
|
|
1375
|
+
markProcessed(filePath, stat.size, 0, stat.mtimeMs);
|
|
1376
|
+
continue;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
const chunks = groupTurnsIntoChunks(turns);
|
|
1380
|
+
const texts = chunks.map((c) => c.text);
|
|
1381
|
+
const batchSize = 32;
|
|
1382
|
+
let fileChunks = 0;
|
|
1383
|
+
|
|
1384
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
1385
|
+
const batch = texts.slice(i, i + batchSize);
|
|
1386
|
+
const batchChunks = chunks.slice(i, i + batchSize);
|
|
1387
|
+
|
|
1388
|
+
try {
|
|
1389
|
+
const vectors = await embedder.embedBatchPassage(batch);
|
|
1390
|
+
const toStore: Array<{text: string; vector: number[]; category: string; scope: string; importance: number; metadata: string}> = [];
|
|
1391
|
+
|
|
1392
|
+
// Dedup + L0 batch
|
|
1393
|
+
const dedupedTexts: string[] = [];
|
|
1394
|
+
const dedupedVectors: number[][] = [];
|
|
1395
|
+
const dedupedChunks: typeof batchChunks = [];
|
|
1396
|
+
|
|
1397
|
+
for (let j = 0; j < batch.length; j++) {
|
|
1398
|
+
const vector = vectors[j];
|
|
1399
|
+
if (!vector || vector.length === 0) {
|
|
1400
|
+
result.chunksSkipped++;
|
|
1401
|
+
continue;
|
|
1402
|
+
}
|
|
1403
|
+
if (!options.noDedup) {
|
|
1404
|
+
const decision = await dedupCheck(store, vector, batch[j], options.llm);
|
|
1405
|
+
recordDedupDecision(result, decision);
|
|
1406
|
+
if (decision.secondaryDeletes?.length) {
|
|
1407
|
+
await executeSecondaryDeletes(store, decision.secondaryDeletes);
|
|
1408
|
+
}
|
|
1409
|
+
if (decision.action === "skip") continue;
|
|
1410
|
+
}
|
|
1411
|
+
dedupedTexts.push(batch[j]);
|
|
1412
|
+
dedupedVectors.push(vector);
|
|
1413
|
+
dedupedChunks.push(batchChunks[j]);
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
if (dedupedTexts.length > 0) {
|
|
1417
|
+
if (!options.llm) {
|
|
1418
|
+
queueForLaterExtraction(
|
|
1419
|
+
dedupedChunks.map(c => ({ text: c.text, scope: `kimi:${c.sessionId.slice(0, 8)}` }))
|
|
1420
|
+
);
|
|
1421
|
+
result.chunksSkipped += dedupedTexts.length;
|
|
1422
|
+
} else {
|
|
1423
|
+
const extractions = await smartExtractBatch(dedupedTexts, options.llm);
|
|
1424
|
+
const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
|
|
1425
|
+
for (let j = 0; j < dedupedTexts.length; j++) {
|
|
1426
|
+
const chunk = dedupedChunks[j];
|
|
1427
|
+
const ext = extractions[j];
|
|
1428
|
+
toStore.push(buildIngestedEntry({
|
|
1429
|
+
source: "kimi",
|
|
1430
|
+
scope: `kimi:${chunk.sessionId.slice(0, 8)}`,
|
|
1431
|
+
text: dedupedTexts[j],
|
|
1432
|
+
vector: dedupedVectors[j],
|
|
1433
|
+
extraction: ext,
|
|
1434
|
+
sessionId: chunk.sessionId,
|
|
1435
|
+
file: basename(filePath),
|
|
1436
|
+
coreSummary: coreSummaries[j],
|
|
1437
|
+
}));
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
if (toStore.length > 0) {
|
|
1443
|
+
await store.storeBatch(toStore);
|
|
1444
|
+
result.chunksIngested += toStore.length;
|
|
1445
|
+
fileChunks += toStore.length;
|
|
1446
|
+
}
|
|
1447
|
+
} catch (err: any) {
|
|
1448
|
+
result.errors.push(`Embedding batch error: ${err.message}`);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
markProcessed(filePath, stat.size, fileChunks, stat.mtimeMs);
|
|
1453
|
+
result.filesProcessed++;
|
|
1454
|
+
|
|
1455
|
+
if ((fi + 1) % 10 === 0 || fi + 1 === total) {
|
|
1456
|
+
console.log(
|
|
1457
|
+
` Kimi: ${fi + 1}/${total} files, ${result.chunksIngested} chunks ingested`,
|
|
1458
|
+
);
|
|
1459
|
+
}
|
|
1460
|
+
} catch (err: any) {
|
|
1461
|
+
result.errors.push(`Error processing ${basename(filePath)}: ${err.message}`);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
return result;
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1193
1468
|
// ============================================================================
|
|
1194
1469
|
// Gemini Session Parser
|
|
1195
1470
|
// ============================================================================
|
package/src/kg-extractor.ts
CHANGED
|
@@ -96,7 +96,13 @@ const DEFAULT_KG_SYSTEM = `You are a knowledge graph extraction assistant. Extra
|
|
|
96
96
|
const DEFAULT_KG_USER_TEMPLATE = `Extract knowledge graph triples from the following text.
|
|
97
97
|
Rules:
|
|
98
98
|
- Use simple predicates: uses, created_by, works_with, is_a, part_of, has, located_in, belongs_to, depends_on, implements, extends, related_to, caused_by, results_in, precedes, follows
|
|
99
|
-
-
|
|
99
|
+
- confidence must be EXACTLY one of these five values (models collapse continuous ranges — pick the one matching the evidence):
|
|
100
|
+
1.0 = stated explicitly in the text
|
|
101
|
+
0.85 = strong inference (clearly implied, never stated)
|
|
102
|
+
0.75 = reasonable inference (requires interpretation)
|
|
103
|
+
0.65 = weak inference (thematic association only)
|
|
104
|
+
0.55 = speculative (surface co-occurrence)
|
|
105
|
+
Never use 0.5 or any value outside this list. If unsure between two tiers, pick the lower.
|
|
100
106
|
- Extract 3-8 triples
|
|
101
107
|
- Return JSON: { "triples": [{ "subject": "", "predicate": "", "object": "", "confidence": 0.0 }] }
|
|
102
108
|
|
package/src/llm-client.ts
CHANGED
|
@@ -106,12 +106,57 @@ export interface LLMConfig {
|
|
|
106
106
|
apiKey: string;
|
|
107
107
|
model: string;
|
|
108
108
|
baseURL: string;
|
|
109
|
+
/** SDK-level retries. Undefined preserves the OpenAI SDK default. */
|
|
110
|
+
maxRetries?: number;
|
|
109
111
|
/** Request timeout in ms (default: 15000) */
|
|
110
112
|
timeoutMs?: number;
|
|
111
113
|
/** Temperature (default: 0.1 for consistency) */
|
|
112
114
|
temperature?: number;
|
|
113
115
|
}
|
|
114
116
|
|
|
117
|
+
export type DetailedChatTokenLimit =
|
|
118
|
+
| { parameter: "max_tokens"; value: number }
|
|
119
|
+
| { parameter: "max_completion_tokens"; value: number };
|
|
120
|
+
|
|
121
|
+
export interface DetailedChatOptions {
|
|
122
|
+
/** Per-request temperature override. Defaults to the client configuration. */
|
|
123
|
+
temperature?: number;
|
|
124
|
+
/** OpenAI-compatible JSON object response mode. */
|
|
125
|
+
responseFormat?: { type: "json_object" };
|
|
126
|
+
/** Provider extension used by Qwen-compatible endpoints. */
|
|
127
|
+
enableThinking?: boolean;
|
|
128
|
+
/** Explicitly records both the token-limit field name and its value. */
|
|
129
|
+
tokenLimit?: DetailedChatTokenLimit;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export interface DetailedChatUsage {
|
|
133
|
+
promptTokens: number;
|
|
134
|
+
completionTokens: number;
|
|
135
|
+
totalTokens: number;
|
|
136
|
+
cachedPromptTokens?: number;
|
|
137
|
+
reasoningTokens?: number;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export interface DetailedChatResult {
|
|
141
|
+
content: string | null;
|
|
142
|
+
finishReason: string | null;
|
|
143
|
+
responseModel: string;
|
|
144
|
+
usage: DetailedChatUsage | null;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
interface OpenAICompatibleLongChatRequest {
|
|
148
|
+
model: string;
|
|
149
|
+
messages: [
|
|
150
|
+
{ role: "system"; content: string },
|
|
151
|
+
{ role: "user"; content: string },
|
|
152
|
+
];
|
|
153
|
+
temperature: number;
|
|
154
|
+
max_tokens?: number;
|
|
155
|
+
max_completion_tokens?: number;
|
|
156
|
+
response_format?: { type: "json_object" };
|
|
157
|
+
enable_thinking?: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
115
160
|
export interface DedupDecision {
|
|
116
161
|
action: "CREATE" | "MERGE" | "SKIP";
|
|
117
162
|
reason: string;
|
|
@@ -190,9 +235,13 @@ export class LLMClient {
|
|
|
190
235
|
|
|
191
236
|
constructor(config: LLMConfig, breakerConfig?: Partial<CircuitBreakerConfig>) {
|
|
192
237
|
const apiKey = resolveEnvVars(config.apiKey);
|
|
238
|
+
if (config.maxRetries !== undefined && (!Number.isInteger(config.maxRetries) || config.maxRetries < 0)) {
|
|
239
|
+
throw new Error("LLM maxRetries must be a non-negative integer");
|
|
240
|
+
}
|
|
193
241
|
this.client = new OpenAI({
|
|
194
242
|
apiKey,
|
|
195
243
|
baseURL: config.baseURL,
|
|
244
|
+
...(config.maxRetries !== undefined ? { maxRetries: config.maxRetries } : {}),
|
|
196
245
|
});
|
|
197
246
|
this.model = config.model;
|
|
198
247
|
this.temperature = config.temperature ?? 0.1;
|
|
@@ -696,29 +745,68 @@ export class LLMClient {
|
|
|
696
745
|
}
|
|
697
746
|
}
|
|
698
747
|
|
|
699
|
-
async
|
|
748
|
+
async chatLongDetailed(
|
|
749
|
+
system: string,
|
|
750
|
+
user: string,
|
|
751
|
+
options: DetailedChatOptions = {},
|
|
752
|
+
): Promise<DetailedChatResult | null> {
|
|
700
753
|
if (!this.breaker.canAttempt()) {
|
|
701
754
|
return null;
|
|
702
755
|
}
|
|
703
756
|
|
|
757
|
+
const tokenLimit = options.tokenLimit ?? { parameter: "max_tokens", value: 2000 };
|
|
758
|
+
if (!Number.isInteger(tokenLimit.value) || tokenLimit.value <= 0) {
|
|
759
|
+
throw new Error("Detailed chat token limit must be a positive integer");
|
|
760
|
+
}
|
|
761
|
+
const temperature = options.temperature ?? this.temperature;
|
|
762
|
+
if (!Number.isFinite(temperature)) {
|
|
763
|
+
throw new Error("Detailed chat temperature must be finite");
|
|
764
|
+
}
|
|
765
|
+
const request: OpenAICompatibleLongChatRequest = {
|
|
766
|
+
model: this.model,
|
|
767
|
+
messages: [
|
|
768
|
+
{ role: "system", content: system },
|
|
769
|
+
{ role: "user", content: user },
|
|
770
|
+
],
|
|
771
|
+
temperature,
|
|
772
|
+
};
|
|
773
|
+
if (tokenLimit.parameter === "max_tokens") {
|
|
774
|
+
request.max_tokens = tokenLimit.value;
|
|
775
|
+
} else {
|
|
776
|
+
request.max_completion_tokens = tokenLimit.value;
|
|
777
|
+
}
|
|
778
|
+
if (options.responseFormat) request.response_format = options.responseFormat;
|
|
779
|
+
if (options.enableThinking !== undefined) request.enable_thinking = options.enableThinking;
|
|
780
|
+
|
|
704
781
|
const controller = new AbortController();
|
|
705
782
|
const timeout = setTimeout(() => controller.abort(), this.timeoutMs * 2);
|
|
706
783
|
|
|
707
784
|
try {
|
|
708
785
|
const response = await this.client.chat.completions.create(
|
|
709
|
-
|
|
710
|
-
model: this.model,
|
|
711
|
-
messages: [
|
|
712
|
-
{ role: "system", content: system },
|
|
713
|
-
{ role: "user", content: user },
|
|
714
|
-
],
|
|
715
|
-
temperature: this.temperature,
|
|
716
|
-
max_tokens: maxTokens,
|
|
717
|
-
},
|
|
786
|
+
request,
|
|
718
787
|
{ signal: controller.signal },
|
|
719
788
|
);
|
|
720
789
|
|
|
721
|
-
const
|
|
790
|
+
const choice = response.choices[0];
|
|
791
|
+
const usage = response.usage
|
|
792
|
+
? {
|
|
793
|
+
promptTokens: response.usage.prompt_tokens,
|
|
794
|
+
completionTokens: response.usage.completion_tokens,
|
|
795
|
+
totalTokens: response.usage.total_tokens,
|
|
796
|
+
...(response.usage.prompt_tokens_details?.cached_tokens !== undefined
|
|
797
|
+
? { cachedPromptTokens: response.usage.prompt_tokens_details.cached_tokens }
|
|
798
|
+
: {}),
|
|
799
|
+
...(response.usage.completion_tokens_details?.reasoning_tokens !== undefined
|
|
800
|
+
? { reasoningTokens: response.usage.completion_tokens_details.reasoning_tokens }
|
|
801
|
+
: {}),
|
|
802
|
+
}
|
|
803
|
+
: null;
|
|
804
|
+
const result: DetailedChatResult = {
|
|
805
|
+
content: choice?.message?.content?.trim() || null,
|
|
806
|
+
finishReason: choice?.finish_reason ?? null,
|
|
807
|
+
responseModel: response.model,
|
|
808
|
+
usage,
|
|
809
|
+
};
|
|
722
810
|
this.breaker.recordSuccess();
|
|
723
811
|
return result;
|
|
724
812
|
} catch (err) {
|
|
@@ -729,6 +817,13 @@ export class LLMClient {
|
|
|
729
817
|
}
|
|
730
818
|
}
|
|
731
819
|
|
|
820
|
+
async chatLong(system: string, user: string, maxTokens = 2000): Promise<string | null> {
|
|
821
|
+
const result = await this.chatLongDetailed(system, user, {
|
|
822
|
+
tokenLimit: { parameter: "max_tokens", value: maxTokens },
|
|
823
|
+
});
|
|
824
|
+
return result?.content ?? null;
|
|
825
|
+
}
|
|
826
|
+
|
|
732
827
|
// --------------------------------------------------------------------------
|
|
733
828
|
// Constructive Retrieval — public wrappers for reconstruction pipeline
|
|
734
829
|
// --------------------------------------------------------------------------
|
package/src/mcp-server.ts
CHANGED
|
@@ -33,6 +33,7 @@ import { registerCoreTools } from "./mcp-tools-core.js";
|
|
|
33
33
|
import { registerAdvancedTools } from "./mcp-tools-advanced.js";
|
|
34
34
|
import { registerGovernanceTools } from "./mcp-tools-governance.js";
|
|
35
35
|
import type { ToolRegistryDeps } from "./mcp-tool-deps.js";
|
|
36
|
+
import { PACKAGE_VERSION } from "./version.js";
|
|
36
37
|
|
|
37
38
|
// ============================================================================
|
|
38
39
|
// Tier Configuration
|
|
@@ -152,7 +153,7 @@ function ensureComponents(): Promise<void> {
|
|
|
152
153
|
|
|
153
154
|
const server = new McpServer({
|
|
154
155
|
name: "recallnest",
|
|
155
|
-
version:
|
|
156
|
+
version: PACKAGE_VERSION,
|
|
156
157
|
});
|
|
157
158
|
|
|
158
159
|
// ============================================================================
|
|
@@ -3,6 +3,7 @@ import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
|
|
|
3
3
|
import { createAuditLogger } from "./audit-log.js";
|
|
4
4
|
import { persistCaseMemory, persistMemoryBatch, persistWorkflowPattern, promoteMemory } from "./capture-engine.js";
|
|
5
5
|
import { autoCapture } from "./capture-heuristic.js";
|
|
6
|
+
import { computeSynthesisUptake } from "./consolidation-engine.js";
|
|
6
7
|
import { runDataCheckup, formatCheckupReport } from "./data-checkup.js";
|
|
7
8
|
import { runDream, formatDreamResult } from "./dream-pipeline.js";
|
|
8
9
|
import { forgetMemory } from "./forget-engine.js";
|
|
@@ -550,6 +551,14 @@ registerTool(
|
|
|
550
551
|
sourceCounts[prefix] = (sourceCounts[prefix] || 0) + count;
|
|
551
552
|
}
|
|
552
553
|
|
|
554
|
+
// Synthesis uptake(Artel 借鉴):升华产物被读回的比例——长期 0% = 升华管线
|
|
555
|
+
// 在产无人消费的内容或读打点断链,是能报警的健康度数字。
|
|
556
|
+
// cap 取 totalCount 自适应库大小(2026-07-23 实测库已 117K,固定 cap 追不上增长)。
|
|
557
|
+
const uptake = await computeSynthesisUptake(store, stats.totalCount + 1_000);
|
|
558
|
+
const uptakeLine = uptake.uptakeRate === null
|
|
559
|
+
? " uptake: n/a (no derived insights found)"
|
|
560
|
+
: ` uptake: ${(uptake.uptakeRate * 100).toFixed(1)}% (${uptake.derivedRead}/${uptake.derivedTotal} read ≥1)`;
|
|
561
|
+
|
|
553
562
|
const lines = [
|
|
554
563
|
`Total entries: ${stats.totalCount}`,
|
|
555
564
|
"",
|
|
@@ -562,6 +571,10 @@ registerTool(
|
|
|
562
571
|
...Object.entries(stats.categoryCounts || {})
|
|
563
572
|
.sort((a, b) => b[1] - a[1])
|
|
564
573
|
.map(([cat, count]) => ` ${cat}: ${count}`),
|
|
574
|
+
"",
|
|
575
|
+
`Synthesis uptake${uptake.truncated ? ` (scan capped at ${uptake.scanned})` : ""}:`,
|
|
576
|
+
` derived insights: ${uptake.derivedTotal}`,
|
|
577
|
+
uptakeLine,
|
|
565
578
|
];
|
|
566
579
|
|
|
567
580
|
return {
|
package/src/mcp-tools-core.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { buildManagedCheckpointObservation, buildManagedResumeObservation } from
|
|
|
16
16
|
import { buildWorkflowObservationRecord } from "./workflow-observation-engine.js";
|
|
17
17
|
import type { ToolRegistryDeps } from "./mcp-tool-deps.js";
|
|
18
18
|
import type { RetrievalResult } from "./retriever.js";
|
|
19
|
+
import { DependsOnSchema } from "./freshness.js";
|
|
19
20
|
|
|
20
21
|
export function registerCoreTools(deps: ToolRegistryDeps): void {
|
|
21
22
|
const { registerTool, getComponents, config, checkpointStore, conflictStore, workflowObservationStore, getKGExtractor, getKGStore, auditLogger } = deps;
|
|
@@ -93,8 +94,9 @@ registerTool(
|
|
|
93
94
|
reliability: z.enum(["direct", "inferred", "hearsay"]).optional(),
|
|
94
95
|
}),
|
|
95
96
|
]).optional().describe("Optional confidence override: number (0-1) or {score, reliability}. Auto-assigned from source if omitted."),
|
|
97
|
+
dependsOn: DependsOnSchema.optional().describe("Optional freshness dependencies (borrowed). Declare what this memory depends on so recall shows a cheap validity verdict (exact/compatible/uncertain/invalid). Items: {kind:'file'|'git-rev', ref, expected?}. file → ref=path, expected=mtime-ms string; git-rev → ref=repo path, expected=commit hash (short ok). expected may be an array: first=exact, rest=compatible set. Omit expected for existence-only checks."),
|
|
96
98
|
},
|
|
97
|
-
async ({ text, category, importance, scope, source, tags, canonicalKey, topicTag, privacyTier, validUntil, eventTime, confidence }) => {
|
|
99
|
+
async ({ text, category, importance, scope, source, tags, canonicalKey, topicTag, privacyTier, validUntil, eventTime, confidence, dependsOn }) => {
|
|
98
100
|
const { store, embedder } = getComponents();
|
|
99
101
|
const kgExtractor = getKGExtractor();
|
|
100
102
|
const stored = await persistMemory({
|
|
@@ -118,6 +120,8 @@ registerTool(
|
|
|
118
120
|
eventTime,
|
|
119
121
|
// F1: Pass confidence override (extracted by persistMemory before Zod parse)
|
|
120
122
|
confidence,
|
|
123
|
+
// Freshness: pass dependsOn (extracted by persistMemory before Zod parse)
|
|
124
|
+
dependsOn,
|
|
121
125
|
});
|
|
122
126
|
|
|
123
127
|
return {
|
package/src/memory-boundaries.ts
CHANGED
|
@@ -61,8 +61,29 @@ export interface IngestBoundaryResolution {
|
|
|
61
61
|
boundary: MemoryBoundaryMetadata;
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
// 在栈各端的 transcript 来源与 scope 前缀。
|
|
65
|
+
// ⚠️ 接入新的 AI 端时必须同步这两处 —— 漏掉的端不会被认成 evidence 层,
|
|
66
|
+
// 其会话碎片既不会被降权,也会被 layer admission 当成 durable 结论放行。
|
|
67
|
+
// (kimi / antigravity 就漏了:kimi 2026-07-19 进栈、antigravity 07-29 进栈,
|
|
68
|
+
// 直到 08-01 才发现库里的 `kimi:` scope 一直被当结论对待。)
|
|
69
|
+
//
|
|
70
|
+
// minis(2026-08-12 加,iPhone 上的 Minis agent,经 iCloud 文件信箱回流对话记录):
|
|
71
|
+
// 这一次是**接入前先补**,不是事后发现——补的动机值得记下来,因为 Minis 比前面几端更危险。
|
|
72
|
+
// 它的 CLI 把 agent 自己的工具调用回传也标成 `type:"user"` / `role:"user"`(实测一场对话
|
|
73
|
+
// 137 条里 69 条 user,其中仅 11 条是真人发言,另外 58 条是 `[Tool result: ...]`)。
|
|
74
|
+
// 所以漏掉它不只是"碎片被当结论",而是**把 shell 输出提炼成用户说过的话**——
|
|
75
|
+
// 例如 `ssh: connect to host ... timed out` 会变成一条"用户说过"的记忆。
|
|
76
|
+
// 导出侧已在上游过滤(只导真人对话),这里是第二道闸:即使某次导出没过滤干净,
|
|
77
|
+
// 走 ingest 的内容也会被正确降权到 evidence 层,而不是当成 durable 结论。
|
|
78
|
+
const TRANSCRIPT_SOURCES = new Set(["cc", "codex", "gemini", "kimi", "antigravity", "minis"]);
|
|
79
|
+
const TRANSCRIPT_SCOPE_PREFIXES = [
|
|
80
|
+
"cc:",
|
|
81
|
+
"codex:",
|
|
82
|
+
"gemini:",
|
|
83
|
+
"kimi:",
|
|
84
|
+
"antigravity:",
|
|
85
|
+
"minis:",
|
|
86
|
+
];
|
|
66
87
|
|
|
67
88
|
export function getConflictPolicyForCategory(category: DurableMemoryCategory): MemoryConflictPolicy {
|
|
68
89
|
return category === "events" || category === "cases"
|
|
@@ -125,6 +125,10 @@ function computeTierAccessSignal(
|
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
// ⚠️ 约束(2026-07-21 通用Wiki上下文记忆Agent方案借鉴审计 P1-4):本函数用 accessCount 参与
|
|
129
|
+
// importance 计算(importance←accessSignal,越常访问 importance 越高的马太逻辑)。当前为孤儿代码——
|
|
130
|
+
// 全树仅测试引用、生产侧零 import。接线前须评估与「usage 只进检索先验、绝不抬 importance/事实置信」
|
|
131
|
+
// 立场的冲突:频次抬 importance 会自增强(越常召回越权威),违反该立场。接线须 Alice 明确批准。
|
|
128
132
|
export function computeRebalancedImportance(
|
|
129
133
|
currentImportance: number,
|
|
130
134
|
tier: MemoryTier,
|
package/src/memory-output.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { extractMultiVectorText } from "./multi-vector.js";
|
|
|
6
6
|
import { parseNarrative } from "./narrative-schema.js";
|
|
7
7
|
import { getConfidence, getConfidenceMetadata } from "./confidence-tracker.js";
|
|
8
8
|
import { estimateTokens } from "./context-collapse-renderer.js";
|
|
9
|
+
import { evaluateEntryFreshness, createFreshnessCache } from "./freshness.js";
|
|
9
10
|
|
|
10
11
|
interface MemoryMetadata {
|
|
11
12
|
source?: string;
|
|
@@ -348,11 +349,15 @@ export function formatBriefResults(
|
|
|
348
349
|
context: { query: string },
|
|
349
350
|
): string {
|
|
350
351
|
if (results.length === 0) return "No results found.";
|
|
352
|
+
const freshnessCache = createFreshnessCache();
|
|
351
353
|
const lines = [`Query: ${context.query}`, `Hits: ${results.length}`, ""];
|
|
352
354
|
for (let i = 0; i < results.length; i++) {
|
|
353
355
|
const r = results[i];
|
|
354
356
|
const excerpt = extractBriefExcerpt(r);
|
|
355
|
-
|
|
357
|
+
// Freshness suffix: only for memories that declared dependsOn (opt-in, cheap check).
|
|
358
|
+
const fresh = evaluateEntryFreshness(r.entry.metadata, freshnessCache);
|
|
359
|
+
const freshTag = fresh ? ` [fresh:${fresh}]` : "";
|
|
360
|
+
lines.push(`#${i + 1} ${r.entry.id.slice(0, 8)} ${(r.score * 100).toFixed(1)}% — ${excerpt}${freshTag}`);
|
|
356
361
|
}
|
|
357
362
|
return lines.join("\n");
|
|
358
363
|
}
|
|
@@ -363,6 +368,7 @@ export function formatFullResults(
|
|
|
363
368
|
): string {
|
|
364
369
|
if (results.length === 0) return "No results found.";
|
|
365
370
|
|
|
371
|
+
const freshnessCache = createFreshnessCache();
|
|
366
372
|
const lines = [
|
|
367
373
|
`Query : ${context.query}`,
|
|
368
374
|
`Profile : ${context.profile}`,
|
|
@@ -380,6 +386,7 @@ export function formatFullResults(
|
|
|
380
386
|
const meta = parseMetadata(results[i].entry);
|
|
381
387
|
const evolution = typeof meta.evolutionStatus === "string" ? meta.evolutionStatus : "-";
|
|
382
388
|
const accessCount = typeof meta.accessCount === "number" ? String(meta.accessCount) : "-";
|
|
389
|
+
const readers = typeof meta.distinctReaderCount === "number" ? String(meta.distinctReaderCount) : "-";
|
|
383
390
|
const importance = results[i].entry.importance.toFixed(2);
|
|
384
391
|
const tags = Array.isArray(meta.tags) ? (meta.tags as string[]).join(", ") : "-";
|
|
385
392
|
// Emotion metadata (from emotion-detector)
|
|
@@ -396,7 +403,10 @@ export function formatFullResults(
|
|
|
396
403
|
const confPart = confMeta
|
|
397
404
|
? ` confidence=${confMeta.score.toFixed(2)}(${confMeta.reliability})`
|
|
398
405
|
: "";
|
|
399
|
-
lines.push(` meta : evolution=${evolution} accessCount=${accessCount} importance=${importance} tags=[${tags}]${confPart}${emotionPart}${narrativePart}`);
|
|
406
|
+
lines.push(` meta : evolution=${evolution} accessCount=${accessCount} readers=${readers} importance=${importance} tags=[${tags}]${confPart}${emotionPart}${narrativePart}`);
|
|
407
|
+
// Freshness: only shown for memories that declared dependsOn (opt-in, cheap check).
|
|
408
|
+
const fresh = evaluateEntryFreshness(results[i].entry.metadata, freshnessCache);
|
|
409
|
+
if (fresh) lines.push(` fresh: ${fresh}`);
|
|
400
410
|
}
|
|
401
411
|
|
|
402
412
|
return lines.join("\n");
|
|
@@ -408,6 +418,7 @@ export function formatSearchResults(
|
|
|
408
418
|
): string {
|
|
409
419
|
if (results.length === 0) return "No results found.";
|
|
410
420
|
|
|
421
|
+
const freshnessCache = createFreshnessCache();
|
|
411
422
|
const lines = [
|
|
412
423
|
`Query : ${context.query}`,
|
|
413
424
|
`Profile : ${context.profile}`,
|
|
@@ -421,6 +432,9 @@ export function formatSearchResults(
|
|
|
421
432
|
const row = buildSearchRow(i, context.query, results[i]);
|
|
422
433
|
lines.push(`${row[0]} ${row[1]} ${row[2]} ${row[3]} ${row[4]} ${row[5]} ${row[6]} ${row[7]} ${row[8]} | ${row[9]}`);
|
|
423
434
|
lines.push(` prov : ${getProvenanceSummary(results[i])}`);
|
|
435
|
+
// Freshness: only shown for memories that declared dependsOn (opt-in, cheap check).
|
|
436
|
+
const fresh = evaluateEntryFreshness(results[i].entry.metadata, freshnessCache);
|
|
437
|
+
if (fresh) lines.push(` fresh: ${fresh}`);
|
|
424
438
|
}
|
|
425
439
|
|
|
426
440
|
return lines.join("\n");
|