recallnest 2.5.4 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +34 -9
- package/README_CN.md +37 -12
- package/package.json +6 -5
- package/src/__tests__/access-tracker-novelty.test.ts +66 -0
- package/src/__tests__/api-server-security.test.ts +102 -0
- package/src/__tests__/belief-history.test.ts +220 -0
- package/src/__tests__/capture-engine.test.ts +93 -3
- package/src/__tests__/category-min-scores.test.ts +83 -0
- package/src/__tests__/cluster-consolidation.test.ts +167 -4
- package/src/__tests__/conflict-store.test.ts +46 -2
- package/src/__tests__/consolidation-engine.test.ts +97 -0
- package/src/__tests__/contracts.test.ts +46 -0
- package/src/__tests__/decay-exemption.test.ts +45 -0
- package/src/__tests__/dream-pipeline.test.ts +464 -5
- package/src/__tests__/freshness.test.ts +186 -0
- package/src/__tests__/hotness-blend.test.ts +207 -0
- package/src/__tests__/llm-client-pivot.test.ts +628 -0
- package/src/__tests__/maintenance-locks.test.ts +15 -0
- package/src/__tests__/memory-boundaries.test.ts +32 -0
- package/src/__tests__/noise-prototype-bank.test.ts +280 -0
- package/src/__tests__/pii-redact.test.ts +19 -0
- package/src/__tests__/pivot-apply.test.ts +373 -0
- package/src/__tests__/pivot-distill-supervisor.test.ts +457 -0
- package/src/__tests__/pivot-distill.test.ts +1178 -0
- package/src/__tests__/preference-matcher.test.ts +54 -2
- package/src/__tests__/pull-from-macbook-sync-only.test.ts +679 -0
- package/src/__tests__/recall-gate.test.ts +266 -0
- package/src/__tests__/resolve-db-path.test.ts +130 -0
- package/src/__tests__/resolve-tier.test.ts +64 -0
- package/src/__tests__/retrieval-eval.test.ts +163 -0
- package/src/__tests__/retriever-lazy-init-dispatch.test.ts +123 -0
- package/src/__tests__/skill-promotion.test.ts +53 -0
- package/src/__tests__/stable-context-length-contract.test.ts +62 -0
- package/src/__tests__/store-read-consistency.test.ts +122 -0
- package/src/__tests__/store-schema-migration.test.ts +87 -0
- package/src/__tests__/store-upsert.test.ts +17 -0
- package/src/__tests__/term-registry-terminal-roster.test.ts +127 -0
- package/src/__tests__/ui-server-security.test.ts +93 -0
- package/src/access-tracker.ts +53 -3
- package/src/adaptive-retrieval.ts +11 -0
- package/src/api-server.ts +9 -2
- package/src/auto-recall.ts +61 -1
- package/src/belief-history.ts +164 -0
- package/src/capture-engine.ts +88 -11
- package/src/cli.ts +109 -14
- package/src/conflict-engine.ts +24 -3
- package/src/consolidation-engine.ts +146 -12
- package/src/context-composer-stable-selection.ts +7 -1
- package/src/data-checkup.ts +1 -1
- package/src/decay-engine.ts +59 -7
- package/src/doctor.ts +4 -2
- package/src/dream-pipeline.ts +359 -11
- package/src/env-config.ts +41 -0
- package/src/eval.ts +154 -11
- package/src/freshness.ts +192 -0
- package/src/http-request-policy.ts +148 -0
- package/src/ingest.ts +277 -2
- package/src/kg-extractor.ts +7 -1
- package/src/llm-client.ts +106 -11
- package/src/mcp-server.ts +2 -1
- package/src/mcp-tools-advanced.ts +13 -0
- package/src/mcp-tools-core.ts +5 -1
- package/src/memory-boundaries.ts +23 -2
- package/src/memory-health-rebalance.ts +4 -0
- package/src/memory-output.ts +16 -2
- package/src/memory-store-port.ts +3 -0
- package/src/noise-prototype-bank.ts +408 -0
- package/src/pii-detector.ts +52 -2
- package/src/pivot-apply.ts +399 -0
- package/src/preference-matcher.ts +37 -3
- package/src/recall-gate.ts +305 -0
- package/src/retrieval-profiles.ts +46 -4
- package/src/retriever.ts +95 -18
- package/src/retroactive-boost.ts +6 -2
- package/src/runtime-config.ts +70 -5
- package/src/skill-promotion.ts +58 -2
- package/src/store.ts +54 -19
- package/src/term-registry.ts +35 -0
- package/src/ui-server.ts +7 -0
- package/src/version.ts +19 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall gate — pre-retrieval triage for auto-recall traffic.
|
|
3
|
+
*
|
|
4
|
+
* Borrowed from memory-lancedb-pro's adaptive-retrieval, reshaped per Codex
|
|
5
|
+
* review (2026-07-17) into a three-way verdict instead of a binary SKIP/FORCE:
|
|
6
|
+
*
|
|
7
|
+
* - "full-recall": message explicitly reaches for memory ("记得/上次/my
|
|
8
|
+
* preference") — always run resume + focused search.
|
|
9
|
+
* - "resume-only": bare continuity nudges ("继续", "下一步", "开始吧") —
|
|
10
|
+
* they need checkpoint/continuity, not an embedding search
|
|
11
|
+
* over their own two characters.
|
|
12
|
+
* - "skip-all": greetings, slash commands, heartbeats, whole-message
|
|
13
|
+
* acks, pure emoji, bare CLI invocations — no memory value.
|
|
14
|
+
* - "pass": no rule matched — default full pipeline.
|
|
15
|
+
*
|
|
16
|
+
* Rollout is governed by RECALLNEST_RECALL_GATE (observe-before-enforce,
|
|
17
|
+
* shared-behaviors §5): "observe" (default) computes the verdict and appends a
|
|
18
|
+
* shadow-log line but never changes behavior; "enforce" acts on the verdict;
|
|
19
|
+
* "off" disables even the shadow logging. Shadow entries carry rule id,
|
|
20
|
+
* message length, and decision — never the message text itself.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { appendFileSync, existsSync, mkdirSync } from "node:fs";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
import { dataDir } from "./env-config.js";
|
|
26
|
+
|
|
27
|
+
export type RecallGateDecision = "full-recall" | "resume-only" | "skip-all" | "pass";
|
|
28
|
+
|
|
29
|
+
export interface RecallGateResult {
|
|
30
|
+
decision: RecallGateDecision;
|
|
31
|
+
/** Stable rule identifier, e.g. "force:memory-cue", "skip:ack", "pass:long". */
|
|
32
|
+
ruleId: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export type RecallGateMode = "observe" | "enforce" | "off";
|
|
36
|
+
|
|
37
|
+
/** RECALLNEST_RECALL_GATE: unset/invalid → "observe" (shadow-only default). */
|
|
38
|
+
export function resolveRecallGateMode(env: NodeJS.ProcessEnv = process.env): RecallGateMode {
|
|
39
|
+
const raw = (env.RECALLNEST_RECALL_GATE || "").trim().toLowerCase();
|
|
40
|
+
if (raw === "enforce") return "enforce";
|
|
41
|
+
if (raw === "off") return "off";
|
|
42
|
+
return "observe";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Short-text rules below only ever fire on messages up to this length; longer
|
|
46
|
+
// messages carry enough content to always deserve the full pipeline.
|
|
47
|
+
const SHORT_TEXT_MAX_CHARS = 80;
|
|
48
|
+
|
|
49
|
+
// --- full-recall cues (substring, any position, force-first per review) ---
|
|
50
|
+
|
|
51
|
+
const FORCE_CUES_ZH = [
|
|
52
|
+
"记得",
|
|
53
|
+
"回忆",
|
|
54
|
+
"想起",
|
|
55
|
+
"上次",
|
|
56
|
+
"上回",
|
|
57
|
+
"之前讨论",
|
|
58
|
+
"之前说",
|
|
59
|
+
"之前聊",
|
|
60
|
+
"之前提",
|
|
61
|
+
"我的偏好",
|
|
62
|
+
"我说过",
|
|
63
|
+
"你说过",
|
|
64
|
+
"说过吗",
|
|
65
|
+
"存过",
|
|
66
|
+
"记过",
|
|
67
|
+
"提醒过",
|
|
68
|
+
];
|
|
69
|
+
|
|
70
|
+
// Word-bounded and hyphen/underscore-guarded: "recall" must not fire on
|
|
71
|
+
// "auto-recall" or "recall-gate.test.ts" (identifiers, not memory intent).
|
|
72
|
+
const FORCE_CUES_EN = [
|
|
73
|
+
/(?<![\w-])remember(?![\w-])/,
|
|
74
|
+
/(?<![\w-])recall(?![\w-])/,
|
|
75
|
+
/(?<![\w-])last time(?![\w-])/,
|
|
76
|
+
/(?<![\w-])previously(?![\w-])/,
|
|
77
|
+
/(?<![\w-])we discussed(?![\w-])/,
|
|
78
|
+
/(?<![\w-])my preferences?(?![\w-])/,
|
|
79
|
+
/(?<![\w-])you said(?![\w-])/,
|
|
80
|
+
/(?<![\w-])i told you(?![\w-])/,
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
// --- resume-only: whole-message continuity nudges ---
|
|
84
|
+
|
|
85
|
+
// Matched against the whole message after trailing particles/punctuation are
|
|
86
|
+
// stripped, so "继续吧" hits but "继续讨论刚才的方案" (real content) passes.
|
|
87
|
+
const RESUME_PHRASES = new Set([
|
|
88
|
+
"继续",
|
|
89
|
+
"接着",
|
|
90
|
+
"接着来",
|
|
91
|
+
"接着做",
|
|
92
|
+
"接着干",
|
|
93
|
+
"接着说",
|
|
94
|
+
"继续做",
|
|
95
|
+
"继续干",
|
|
96
|
+
"下一步",
|
|
97
|
+
"开始",
|
|
98
|
+
"开工",
|
|
99
|
+
"走起",
|
|
100
|
+
"往下",
|
|
101
|
+
"往下走",
|
|
102
|
+
"continue",
|
|
103
|
+
"go on",
|
|
104
|
+
"next",
|
|
105
|
+
"next step",
|
|
106
|
+
"resume",
|
|
107
|
+
"keep going",
|
|
108
|
+
"carry on",
|
|
109
|
+
"proceed",
|
|
110
|
+
"pick up where we left off",
|
|
111
|
+
]);
|
|
112
|
+
|
|
113
|
+
// --- skip-all: whole-message greetings & acks ---
|
|
114
|
+
|
|
115
|
+
const GREETINGS = new Set([
|
|
116
|
+
"你好",
|
|
117
|
+
"您好",
|
|
118
|
+
"哈喽",
|
|
119
|
+
"嗨",
|
|
120
|
+
"早",
|
|
121
|
+
"早上好",
|
|
122
|
+
"早安",
|
|
123
|
+
"中午好",
|
|
124
|
+
"下午好",
|
|
125
|
+
"晚上好",
|
|
126
|
+
"晚安",
|
|
127
|
+
"在吗",
|
|
128
|
+
"在不在",
|
|
129
|
+
"在么",
|
|
130
|
+
"hi",
|
|
131
|
+
"hello",
|
|
132
|
+
"hey",
|
|
133
|
+
"yo",
|
|
134
|
+
"good morning",
|
|
135
|
+
"good night",
|
|
136
|
+
"good evening",
|
|
137
|
+
"morning",
|
|
138
|
+
"gm",
|
|
139
|
+
"gn",
|
|
140
|
+
]);
|
|
141
|
+
|
|
142
|
+
const ACK_WORDS = new Set([
|
|
143
|
+
"好的",
|
|
144
|
+
"好",
|
|
145
|
+
"好嘞",
|
|
146
|
+
"好滴",
|
|
147
|
+
"好呀",
|
|
148
|
+
"嗯",
|
|
149
|
+
"嗯嗯",
|
|
150
|
+
"哦",
|
|
151
|
+
"噢",
|
|
152
|
+
"喔",
|
|
153
|
+
"行",
|
|
154
|
+
"可以",
|
|
155
|
+
"可",
|
|
156
|
+
"没问题",
|
|
157
|
+
"没事",
|
|
158
|
+
"收到",
|
|
159
|
+
"明白",
|
|
160
|
+
"了解",
|
|
161
|
+
"知道了",
|
|
162
|
+
"懂了",
|
|
163
|
+
"对",
|
|
164
|
+
"对的",
|
|
165
|
+
"是",
|
|
166
|
+
"是的",
|
|
167
|
+
"不用",
|
|
168
|
+
"不用了",
|
|
169
|
+
"谢谢",
|
|
170
|
+
"谢了",
|
|
171
|
+
"多谢",
|
|
172
|
+
"辛苦了",
|
|
173
|
+
"辛苦",
|
|
174
|
+
"牛",
|
|
175
|
+
"赞",
|
|
176
|
+
"棒",
|
|
177
|
+
"妙",
|
|
178
|
+
"ok",
|
|
179
|
+
"okay",
|
|
180
|
+
"k",
|
|
181
|
+
"kk",
|
|
182
|
+
"yes",
|
|
183
|
+
"yep",
|
|
184
|
+
"yeah",
|
|
185
|
+
"no",
|
|
186
|
+
"nope",
|
|
187
|
+
"sure",
|
|
188
|
+
"thanks",
|
|
189
|
+
"thank you",
|
|
190
|
+
"thx",
|
|
191
|
+
"ty",
|
|
192
|
+
"cool",
|
|
193
|
+
"nice",
|
|
194
|
+
"great",
|
|
195
|
+
"done",
|
|
196
|
+
"got it",
|
|
197
|
+
"will do",
|
|
198
|
+
]);
|
|
199
|
+
|
|
200
|
+
// Bare CLI invocations ("git push", "npm test"). A leading tool name alone is
|
|
201
|
+
// not enough — the rest must look like arguments, not prose: any CJK character
|
|
202
|
+
// or question mark ("git 为什么失败", "why does npm test hang?") disqualifies.
|
|
203
|
+
const CLI_SHAPE =
|
|
204
|
+
/^(git|npm|npx|bun|bunx|yarn|pnpm|node|python3?|pip3?|cargo|make|docker|kubectl|brew|ssh|ls|cd|cat|grep|curl)(\s+[^㐀-鿿??]*)?$/i;
|
|
205
|
+
|
|
206
|
+
// Emoji message test: strip whitespace, then require every remaining unit to
|
|
207
|
+
// be pictographic or an emoji modifier. \p{Extended_Pictographic} deliberately
|
|
208
|
+
// excludes bare digits/#/* (they only render as emoji inside keycap sequences
|
|
209
|
+
// with U+FE0F+U+20E3), so "42" and "#1" fall through to "pass".
|
|
210
|
+
const EMOJI_UNIT =
|
|
211
|
+
/^(?:\p{Extended_Pictographic}|\p{Regional_Indicator}|[\u{1F3FB}-\u{1F3FF}\u{FE0F}\u{200D}\u{20E3}])+$/u;
|
|
212
|
+
|
|
213
|
+
// Trailing tone particles + punctuation tolerated on whole-message matches
|
|
214
|
+
// ("好的吧。", "继续吧!", "开始咯~").
|
|
215
|
+
const TRAILING_FLUFF = /[吧呗啊呀哦嘛喽咯哈嘞捏诶欸了]*[\s。,,.!!??~〜…‥·、;;::'"'"()()]*$/u;
|
|
216
|
+
|
|
217
|
+
function stripTrailingFluff(text: string): string {
|
|
218
|
+
return text.replace(TRAILING_FLUFF, "");
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function normalize(message: string): string {
|
|
222
|
+
return message.replace(/\s+/g, " ").trim();
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function isAckMessage(lower: string): boolean {
|
|
226
|
+
const stripped = stripTrailingFluff(lower);
|
|
227
|
+
if (!stripped) return false;
|
|
228
|
+
if (ACK_WORDS.has(stripped)) return true;
|
|
229
|
+
// Multi-token acks ("好的 谢谢", "ok thanks"): every token must be an ack word.
|
|
230
|
+
const tokens = stripped.split(/[\s,,。!!~]+/).filter(Boolean);
|
|
231
|
+
return tokens.length > 1 && tokens.every((t) => ACK_WORDS.has(stripTrailingFluff(t)));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Classify one auto-recall message. Pure and deterministic — call order is
|
|
236
|
+
* force cues → length guard → structural skips → whole-message skips →
|
|
237
|
+
* resume nudges → default pass.
|
|
238
|
+
*/
|
|
239
|
+
export function classifyRecallGate(message: string): RecallGateResult {
|
|
240
|
+
const text = normalize(message);
|
|
241
|
+
if (!text) return { decision: "skip-all", ruleId: "skip:empty" };
|
|
242
|
+
const lower = text.toLowerCase();
|
|
243
|
+
|
|
244
|
+
// 1. Force cues outrank every short-text rule ("可以按上次方案做" must not
|
|
245
|
+
// be swallowed by the ack rule).
|
|
246
|
+
for (const cue of FORCE_CUES_ZH) {
|
|
247
|
+
if (text.includes(cue)) return { decision: "full-recall", ruleId: "force:memory-cue-zh" };
|
|
248
|
+
}
|
|
249
|
+
for (const cue of FORCE_CUES_EN) {
|
|
250
|
+
if (cue.test(lower)) return { decision: "full-recall", ruleId: "force:memory-cue-en" };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// 2. Long messages always deserve the full pipeline.
|
|
254
|
+
if (text.length > SHORT_TEXT_MAX_CHARS) return { decision: "pass", ruleId: "pass:long" };
|
|
255
|
+
|
|
256
|
+
// 3. Structural skips.
|
|
257
|
+
if (/^\/\S+/.test(text)) return { decision: "skip-all", ruleId: "skip:slash" };
|
|
258
|
+
if (/^HEARTBEAT/i.test(text)) return { decision: "skip-all", ruleId: "skip:heartbeat" };
|
|
259
|
+
if (EMOJI_UNIT.test(text.replace(/\s+/g, ""))) {
|
|
260
|
+
return { decision: "skip-all", ruleId: "skip:emoji" };
|
|
261
|
+
}
|
|
262
|
+
if (text.length <= 60 && CLI_SHAPE.test(text)) {
|
|
263
|
+
return { decision: "skip-all", ruleId: "skip:cli" };
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// 4. Whole-message conversational skips.
|
|
267
|
+
const strippedLower = stripTrailingFluff(lower);
|
|
268
|
+
if (GREETINGS.has(strippedLower)) return { decision: "skip-all", ruleId: "skip:greeting" };
|
|
269
|
+
if (isAckMessage(lower)) return { decision: "skip-all", ruleId: "skip:ack" };
|
|
270
|
+
|
|
271
|
+
// 5. Continuity nudges → resume context without a focused search.
|
|
272
|
+
if (RESUME_PHRASES.has(strippedLower)) {
|
|
273
|
+
return { decision: "resume-only", ruleId: "resume:continuity" };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
return { decision: "pass", ruleId: "pass:default" };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// --- shadow log (observe-before-enforce evidence trail) ---
|
|
280
|
+
|
|
281
|
+
export interface RecallGateShadowEntry {
|
|
282
|
+
ts: string;
|
|
283
|
+
decision: RecallGateDecision;
|
|
284
|
+
ruleId: string;
|
|
285
|
+
msgLen: number;
|
|
286
|
+
mode: Exclude<RecallGateMode, "off">;
|
|
287
|
+
/** Caller-provided origin, e.g. "api:/v1/auto-recall". Never message text. */
|
|
288
|
+
source?: string;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
export const RECALL_GATE_SHADOW_FILE = "recall-gate-shadow.jsonl";
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Append one shadow-log line under dataDir(). Must never block or throw —
|
|
295
|
+
* gate telemetry failing is strictly better than recall failing.
|
|
296
|
+
*/
|
|
297
|
+
export function logRecallGateShadow(entry: RecallGateShadowEntry): void {
|
|
298
|
+
try {
|
|
299
|
+
const dir = dataDir();
|
|
300
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
301
|
+
appendFileSync(join(dir, RECALL_GATE_SHADOW_FILE), `${JSON.stringify(entry)}\n`, "utf8");
|
|
302
|
+
} catch {
|
|
303
|
+
// Shadow logging is best-effort by design.
|
|
304
|
+
}
|
|
305
|
+
}
|
|
@@ -1,7 +1,46 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_CATEGORY_MIN_SCORES,
|
|
3
|
+
DEFAULT_RETRIEVAL_CONFIG,
|
|
4
|
+
type RetrievalConfig,
|
|
5
|
+
} from "./retriever.js";
|
|
2
6
|
|
|
3
7
|
export type RetrievalProfileName = "default" | "writing" | "debug" | "fact-check";
|
|
4
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Scale the default per-category thresholds by how loose/tight a profile is overall.
|
|
11
|
+
*
|
|
12
|
+
* Why this exists: a profile used to override `hardMinScore` only, while
|
|
13
|
+
* `Retriever.minScoreFor()` reads `categoryMinScores` first and only falls back to
|
|
14
|
+
* `hardMinScore` for categories not listed. Since the default map lists every
|
|
15
|
+
* category, a profile's looser/tighter intent never reached category admission at
|
|
16
|
+
* all — picking `debug` ("I want episodic detail") still admitted `events` at 0.35
|
|
17
|
+
* and `patterns` at 0.45, exactly as `default` would.
|
|
18
|
+
*
|
|
19
|
+
* IMPORTANT: the returned map must stay complete. A partial map would silently send
|
|
20
|
+
* unlisted categories to `hardMinScore` instead of their default threshold.
|
|
21
|
+
*
|
|
22
|
+
* simplified: proportional scaling only — the *relative* ordering across categories
|
|
23
|
+
* is inherited from DEFAULT_CATEGORY_MIN_SCORES unchanged. Re-designing that ordering
|
|
24
|
+
* per profile (e.g. `debug` favouring episodic categories, `writing` favouring
|
|
25
|
+
* semantic ones) is the better end state but needs canary evidence to pick numbers;
|
|
26
|
+
* do that via `bun run eval:canary` before touching the ratios here.
|
|
27
|
+
*/
|
|
28
|
+
function scaledCategoryMinScores(profileHardMinScore: number): Record<string, number> {
|
|
29
|
+
const factor = profileHardMinScore / DEFAULT_RETRIEVAL_CONFIG.hardMinScore;
|
|
30
|
+
return Object.fromEntries(
|
|
31
|
+
Object.entries(DEFAULT_CATEGORY_MIN_SCORES).map(([category, minScore]) => [
|
|
32
|
+
category,
|
|
33
|
+
Math.round(minScore * factor * 1000) / 1000,
|
|
34
|
+
]),
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Kept as named constants so each profile's `hardMinScore` and its scaled category
|
|
39
|
+
// thresholds cannot drift apart.
|
|
40
|
+
const WRITING_HARD_MIN_SCORE = 0.24;
|
|
41
|
+
const DEBUG_HARD_MIN_SCORE = 0.34;
|
|
42
|
+
const FACT_CHECK_HARD_MIN_SCORE = 0.38;
|
|
43
|
+
|
|
5
44
|
export interface RetrievalProfileDefinition {
|
|
6
45
|
name: RetrievalProfileName;
|
|
7
46
|
label: string;
|
|
@@ -27,7 +66,8 @@ export const RETRIEVAL_PROFILES: Record<RetrievalProfileName, RetrievalProfileDe
|
|
|
27
66
|
recencyHalfLifeDays: 90,
|
|
28
67
|
recencyWeight: 0.06,
|
|
29
68
|
timeDecayHalfLifeDays: 365,
|
|
30
|
-
hardMinScore:
|
|
69
|
+
hardMinScore: WRITING_HARD_MIN_SCORE,
|
|
70
|
+
categoryMinScores: scaledCategoryMinScores(WRITING_HARD_MIN_SCORE),
|
|
31
71
|
lengthNormAnchor: 800,
|
|
32
72
|
},
|
|
33
73
|
},
|
|
@@ -42,7 +82,8 @@ export const RETRIEVAL_PROFILES: Record<RetrievalProfileName, RetrievalProfileDe
|
|
|
42
82
|
recencyHalfLifeDays: 7,
|
|
43
83
|
recencyWeight: 0.15,
|
|
44
84
|
timeDecayHalfLifeDays: 30,
|
|
45
|
-
hardMinScore:
|
|
85
|
+
hardMinScore: DEBUG_HARD_MIN_SCORE,
|
|
86
|
+
categoryMinScores: scaledCategoryMinScores(DEBUG_HARD_MIN_SCORE),
|
|
46
87
|
lengthNormAnchor: 420,
|
|
47
88
|
},
|
|
48
89
|
},
|
|
@@ -57,7 +98,8 @@ export const RETRIEVAL_PROFILES: Record<RetrievalProfileName, RetrievalProfileDe
|
|
|
57
98
|
recencyHalfLifeDays: 21,
|
|
58
99
|
recencyWeight: 0.08,
|
|
59
100
|
timeDecayHalfLifeDays: 45,
|
|
60
|
-
hardMinScore:
|
|
101
|
+
hardMinScore: FACT_CHECK_HARD_MIN_SCORE,
|
|
102
|
+
categoryMinScores: scaledCategoryMinScores(FACT_CHECK_HARD_MIN_SCORE),
|
|
61
103
|
lengthNormAnchor: 360,
|
|
62
104
|
},
|
|
63
105
|
},
|
package/src/retriever.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { expandQuery } from "./query-expander.js";
|
|
|
14
14
|
import { detectLang, tokenizeFts } from "./language-hook.js";
|
|
15
15
|
import { type AccessTracker, computeHotnessScore, parseAccessMetadata } from "./access-tracker.js";
|
|
16
16
|
import { weibullDecay, resolveTier, isDecayExempt, adjustHalfLifeForEmotion, computeArousalBoost } from "./decay-engine.js";
|
|
17
|
-
import { logWarn } from "./stderr-log.js";
|
|
17
|
+
import { logInfo, logWarn } from "./stderr-log.js";
|
|
18
18
|
import { extractBoundaryMetadata, isDurableMemoryScope, isTranscriptScope } from "./memory-boundaries.js";
|
|
19
19
|
import type { TraceCollector } from "./retrieval-trace.js";
|
|
20
20
|
import { extractTopicTag } from "./topic-tag.js";
|
|
@@ -107,6 +107,7 @@ export interface RetrievalConfig {
|
|
|
107
107
|
/**
|
|
108
108
|
* Hotness blend weight. Blends access-frequency hotness into final score.
|
|
109
109
|
* Formula: final = score * (1-alpha) + hotness * alpha.
|
|
110
|
+
* Zero-access rows keep their score unchanged (bonus-only cold start).
|
|
110
111
|
* 0 = disabled (default), 0.15 = recommended.
|
|
111
112
|
*/
|
|
112
113
|
hotnessWeight: number;
|
|
@@ -240,7 +241,16 @@ export const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = {
|
|
|
240
241
|
filterNoise: true,
|
|
241
242
|
rerankModel: "jina-reranker-v3",
|
|
242
243
|
rerankEndpoint: "https://api.jina.ai/v1/rerank",
|
|
243
|
-
|
|
244
|
+
// 2026-08-06:500 → 800。不是拍的——writing profile 早就在用 800
|
|
245
|
+
// (retrieval-profiles.ts),这里只是把一个已在生产跑着的值推广到 default。
|
|
246
|
+
// 实测(eval/ghost-scan.ts,14,949 条 active 全库快照):结构性幽灵
|
|
247
|
+
// 775 → 245(5.18% → 1.64%),救回 530 条,中位长度 1,356 字符那批正落在
|
|
248
|
+
// 打折最重的区间。挤占效应见 eval/lengthnorm-shadow.ts:20 条真实 query
|
|
249
|
+
// 里 14 条 top-6 完全不变,6 条掉出在 limit 放大到 20 后全部复现
|
|
250
|
+
// ——即没有任何条目被删掉,只是挤出窗口,而那 6 条的重排来自
|
|
251
|
+
// applySourceDiversity + MMR(候选数 ≤ limit 时它根本不轮转,
|
|
252
|
+
// anchor 放大让更多候选过阈值才把这个既有行为激活),与本参数无关。
|
|
253
|
+
lengthNormAnchor: 800,
|
|
244
254
|
hardMinScore: 0.35,
|
|
245
255
|
timeDecayHalfLifeDays: 60,
|
|
246
256
|
hotnessWeight: 0,
|
|
@@ -253,6 +263,14 @@ export const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = {
|
|
|
253
263
|
* (mem0/recallnest/GitHub),避免日常语义 query 误触发 BM25 噪声。
|
|
254
264
|
*/
|
|
255
265
|
export function hasExactToken(query: string): boolean {
|
|
266
|
+
// 【为什么中文 query 不触发本函数 —— 这是实测后的决定,不是 ASCII-only 的遗漏】
|
|
267
|
+
// 2026-08-01 用临时开关对 5 条真实中文 query 做过 A/B:强制让中文触发 BM25 补偿后,
|
|
268
|
+
// 3/4 的 query 零新增(BM25 捞回的条目 vector 早已召回,被 vecIds 去重滤掉),
|
|
269
|
+
// 唯一有新增的那次补进 3 条、其中 1 条是当天无关对话,且补偿结果是**插在队首**的。
|
|
270
|
+
// 中文分词本身没问题(babel-memory 已注册,detectLang 正确返回 zh,
|
|
271
|
+
// "双机契约" → "双机 契约"),所以这不是 tokenizer 问题 ——
|
|
272
|
+
// 是 BM25 对中文**语义类** query 的边际收益本就接近零,放开只会拿噪声占前三名。
|
|
273
|
+
// 若将来要重开,先复现那组 A/B,别只凭"中文没走 BM25"这个现象下判断。
|
|
256
274
|
for (const t of query.split(/[\s,。、;:()()【】「」"'`??!!]+/)) {
|
|
257
275
|
if (t.length < 5) continue;
|
|
258
276
|
// 全大写 + 下划线常量(必须含下划线,避免 JSON/HTTP/GitHub 等普通缩写/驼峰误触发)
|
|
@@ -780,6 +798,16 @@ export class MemoryRetriever {
|
|
|
780
798
|
|
|
781
799
|
let results: RetrievalResult[];
|
|
782
800
|
|
|
801
|
+
// 分派前必须先让 store 完成惰性初始化,否则下面读到的 hasFtsSupport 是初始化前的
|
|
802
|
+
// false,hybrid 模式的**首次**检索会静默落回 vector 路径(同一实例第二次才正常)。
|
|
803
|
+
// 该缺陷自 c10607a 初始版本就在,长期未暴露有两个原因:生产 mode 一直是 "vector",
|
|
804
|
+
// 第一个条件短路、根本读不到第二个;而 eval 的 createFreshEvalComponentsFactory 每
|
|
805
|
+
// case 新建组件,每个 case 都停在"首次",于是 eval 从未测到过 hybrid 路径。
|
|
806
|
+
// 注:supplementExactTokenWithBm25 那里的顺序依赖是被意识到的(见其注释"hasFtsSupport
|
|
807
|
+
// 在 vectorOnly 触发 store init 后为真"),漏的只是这个分派点。
|
|
808
|
+
// `?.` 是必需的:测试大量使用鸭子类型的 fake store,它们没有 ready()。
|
|
809
|
+
await this.store.ready?.();
|
|
810
|
+
|
|
783
811
|
// For vector-only mode, use legacy behavior
|
|
784
812
|
if (this.config.mode === "vector" || !this.store.hasFtsSupport) {
|
|
785
813
|
results = await this.vectorOnlyRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace);
|
|
@@ -1262,10 +1290,14 @@ export class MemoryRetriever {
|
|
|
1262
1290
|
? hardFiltered
|
|
1263
1291
|
: hardFiltered.filter(r => isActiveMemory(r.entry.metadata));
|
|
1264
1292
|
|
|
1265
|
-
|
|
1293
|
+
// LA-1: 资格层准入 —— 默认让 durable 层回答,evidence 留给 deja / 显式溯源。
|
|
1294
|
+
// 放在各类 dedup 之前:先定"谁有资格进候选集",再谈多样性与去重。
|
|
1295
|
+
const layerAdmitted = this.applyLayerAdmission(evolutionFiltered, trace);
|
|
1296
|
+
|
|
1297
|
+
trace?.startStage("noise_filter", layerAdmitted.length);
|
|
1266
1298
|
const denoised = this.config.filterNoise
|
|
1267
|
-
? filterNoise(
|
|
1268
|
-
:
|
|
1299
|
+
? filterNoise(layerAdmitted, r => r.entry.text)
|
|
1300
|
+
: layerAdmitted;
|
|
1269
1301
|
trace?.endStage(denoised.length, denoised.map(r => r.score));
|
|
1270
1302
|
|
|
1271
1303
|
// RIF: demote near-duplicate weak results to improve diversity
|
|
@@ -1766,6 +1798,49 @@ export class MemoryRetriever {
|
|
|
1766
1798
|
return weighted.sort((a, b) => b.score - a.score);
|
|
1767
1799
|
}
|
|
1768
1800
|
|
|
1801
|
+
/** LA-1: 一条结果是否属于 evidence 层(transcript 碎片)。判据与 applyBoundaryWeight 保持一致。 */
|
|
1802
|
+
private isEvidenceLayer(r: RetrievalResult): boolean {
|
|
1803
|
+
const boundary = extractBoundaryMetadata(r.entry.metadata);
|
|
1804
|
+
if (boundary) return boundary.layer === "evidence";
|
|
1805
|
+
return isTranscriptScope(r.entry.scope);
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
/**
|
|
1809
|
+
* LA-1: 资格层准入 —— 决定"哪一层来回答这个 query",而非调整分数。
|
|
1810
|
+
*
|
|
1811
|
+
* 与 applyBoundaryWeight 的区别是本方法存在的全部理由:那个改分数(排序层),
|
|
1812
|
+
* 会让来源盖过相关性(`retriever-boundary-weight.test.ts` 明确禁止这点);
|
|
1813
|
+
* 本方法只挑选候选集,层内相关性排序原封不动。
|
|
1814
|
+
*
|
|
1815
|
+
* durable 命中不足 `layerAdmissionMin` 时回退到全量候选池 —— 新主题在结论层
|
|
1816
|
+
* 往往是空的,宁缺毋滥不能变成什么都不给。
|
|
1817
|
+
*/
|
|
1818
|
+
private applyLayerAdmission(
|
|
1819
|
+
results: RetrievalResult[],
|
|
1820
|
+
trace?: TraceCollector,
|
|
1821
|
+
): RetrievalResult[] {
|
|
1822
|
+
const mode = envConfig.layerAdmission();
|
|
1823
|
+
if (mode === "off" || results.length === 0) return results;
|
|
1824
|
+
|
|
1825
|
+
const durable = results.filter(r => !this.isEvidenceLayer(r));
|
|
1826
|
+
const dropped = results.length - durable.length;
|
|
1827
|
+
const thin = durable.length < envConfig.layerAdmissionMin();
|
|
1828
|
+
|
|
1829
|
+
if (mode === "observe") {
|
|
1830
|
+
logInfo(
|
|
1831
|
+
`[layer-admission] observe: durable=${durable.length} evidence=${dropped} ` +
|
|
1832
|
+
`total=${results.length} action=${thin ? "fallback(keep-all)" : "would-drop-evidence"}`,
|
|
1833
|
+
);
|
|
1834
|
+
return results;
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
if (thin) return results;
|
|
1838
|
+
|
|
1839
|
+
trace?.startStage("layer_admission", results.length);
|
|
1840
|
+
trace?.endStage(durable.length, durable.map(r => r.score));
|
|
1841
|
+
return durable;
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1769
1844
|
private applyBoundaryWeight(results: RetrievalResult[]): RetrievalResult[] {
|
|
1770
1845
|
const weighted = results.map(r => {
|
|
1771
1846
|
const boundary = extractBoundaryMetadata(r.entry.metadata);
|
|
@@ -1836,8 +1911,8 @@ export class MemoryRetriever {
|
|
|
1836
1911
|
|
|
1837
1912
|
const now = Date.now();
|
|
1838
1913
|
const decayed = results.map(r => {
|
|
1839
|
-
// HP-7: Exempt core/pinned/recently-accessed from decay
|
|
1840
|
-
if (isDecayExempt(r.entry.metadata, r.entry.importance)) return r;
|
|
1914
|
+
// HP-7: Exempt core/pinned/recently-accessed (and procedural categories) from decay
|
|
1915
|
+
if (isDecayExempt(r.entry.metadata, r.entry.importance, r.entry.category)) return r;
|
|
1841
1916
|
|
|
1842
1917
|
const ts = (r.entry.timestamp && r.entry.timestamp > 0) ? r.entry.timestamp : now;
|
|
1843
1918
|
const ageDays = (now - ts) / 86_400_000;
|
|
@@ -1854,7 +1929,7 @@ export class MemoryRetriever {
|
|
|
1854
1929
|
}
|
|
1855
1930
|
|
|
1856
1931
|
// Resolve tier from metadata (defaults to peripheral)
|
|
1857
|
-
const tier = resolveTier(r.entry.metadata);
|
|
1932
|
+
const tier = resolveTier(r.entry.metadata, r.entry.importance);
|
|
1858
1933
|
|
|
1859
1934
|
// Weibull decay with tier-specific shape and floor
|
|
1860
1935
|
const factor = weibullDecay(ageDays, halfLife, tier);
|
|
@@ -1869,14 +1944,6 @@ export class MemoryRetriever {
|
|
|
1869
1944
|
return decayed.sort((a, b) => b.score - a.score);
|
|
1870
1945
|
}
|
|
1871
1946
|
|
|
1872
|
-
/**
|
|
1873
|
-
* Blend access-frequency hotness into retrieval scores.
|
|
1874
|
-
*
|
|
1875
|
-
* Formula: final = score * (1 - alpha) + hotness * alpha
|
|
1876
|
-
* where hotness = sigmoid(log1p(accessCount)) * exp(-decayRate * ageDays)
|
|
1877
|
-
*
|
|
1878
|
-
* No-op when hotnessWeight is 0 or AccessTracker is absent.
|
|
1879
|
-
*/
|
|
1880
1947
|
/**
|
|
1881
1948
|
* B-1/E-1: Evolution decay blend — factor in composite decay score
|
|
1882
1949
|
* (time × frequency × importance) as a scoring signal.
|
|
@@ -1885,8 +1952,8 @@ export class MemoryRetriever {
|
|
|
1885
1952
|
private applyEvolutionDecayBlend(results: RetrievalResult[]): RetrievalResult[] {
|
|
1886
1953
|
const EVOLUTION_BLEND_WEIGHT = 0.10;
|
|
1887
1954
|
return results.map(r => {
|
|
1888
|
-
// HP-7: Exempt entries get max decay score (1.0) → no penalty
|
|
1889
|
-
if (isDecayExempt(r.entry.metadata, r.entry.importance)) {
|
|
1955
|
+
// HP-7: Exempt entries (incl. procedural categories) get max decay score (1.0) → no penalty
|
|
1956
|
+
if (isDecayExempt(r.entry.metadata, r.entry.importance, r.entry.category)) {
|
|
1890
1957
|
return {
|
|
1891
1958
|
...r,
|
|
1892
1959
|
score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + 1.0 * EVOLUTION_BLEND_WEIGHT, 0),
|
|
@@ -2087,6 +2154,15 @@ export class MemoryRetriever {
|
|
|
2087
2154
|
return [...results, ...siblings].sort((a, b) => b.score - a.score);
|
|
2088
2155
|
}
|
|
2089
2156
|
|
|
2157
|
+
/**
|
|
2158
|
+
* Blend access-frequency hotness into retrieval scores.
|
|
2159
|
+
*
|
|
2160
|
+
* Formula: final = score * (1 - alpha) + hotness * alpha
|
|
2161
|
+
* where hotness = sigmoid(log1p(accessCount)) * exp(-decayRate * ageDays)
|
|
2162
|
+
* Zero-access rows keep their incoming score; accessed rows retain this formula.
|
|
2163
|
+
*
|
|
2164
|
+
* No-op when hotnessWeight is 0 or AccessTracker is absent.
|
|
2165
|
+
*/
|
|
2090
2166
|
private applyHotnessBlend(results: RetrievalResult[]): RetrievalResult[] {
|
|
2091
2167
|
const raw = this.config.hotnessWeight;
|
|
2092
2168
|
const alpha = Math.min(1, Math.max(0, Number.isFinite(raw) ? raw : 0));
|
|
@@ -2098,6 +2174,7 @@ export class MemoryRetriever {
|
|
|
2098
2174
|
accessCount,
|
|
2099
2175
|
lastAccessedAt || r.entry.timestamp,
|
|
2100
2176
|
);
|
|
2177
|
+
if (accessCount === 0) return r;
|
|
2101
2178
|
return {
|
|
2102
2179
|
...r,
|
|
2103
2180
|
score: clamp01(r.score * (1 - alpha) + hotness * alpha, 0),
|
package/src/retroactive-boost.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import type { MemoryStore, MemoryEntry } from "./store.js";
|
|
24
|
+
import { isActiveMemory } from "./memory-evolution.js";
|
|
24
25
|
|
|
25
26
|
// ---------------------------------------------------------------------------
|
|
26
27
|
// Config
|
|
@@ -85,12 +86,15 @@ export async function retroactiveBoost(
|
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
// Find related entries in the same scope
|
|
88
|
-
|
|
89
|
+
// Superseded belief-history rows come back from vectorSearch too. Boosting one is wasted
|
|
90
|
+
// — they are excluded from retrieval anyway — and it burns maxBoostPerTrigger budget that
|
|
91
|
+
// live entries should be getting.
|
|
92
|
+
const candidates = (await store.vectorSearch(
|
|
89
93
|
newEntry.vector,
|
|
90
94
|
config.maxBoostPerTrigger * 3, // fetch more, filter down
|
|
91
95
|
config.similarityThreshold,
|
|
92
96
|
[newEntry.scope],
|
|
93
|
-
);
|
|
97
|
+
)).filter((candidate) => isActiveMemory(candidate.entry.metadata));
|
|
94
98
|
|
|
95
99
|
const now = Date.now();
|
|
96
100
|
const minAgeMs = config.minAgeDays * 86_400_000;
|