recallnest 1.4.1 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +7 -3
  3. package/README.md +202 -130
  4. package/README_CN.md +263 -190
  5. package/bin/recallnest.mjs +62 -0
  6. package/config.json +55 -0
  7. package/daily-recall.sh +23 -0
  8. package/integrations/setup-all.sh +31 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +30 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/launchagent-stderr.log +0 -0
  14. package/logs/launchagent-stdout.log +0 -0
  15. package/package.json +10 -3
  16. package/src/access-tracker.ts +3 -2
  17. package/src/activity-counter.ts +109 -0
  18. package/src/adaptive-retrieval.ts +6 -5
  19. package/src/admission-control.ts +163 -0
  20. package/src/anchor-generator.ts +80 -0
  21. package/src/api-server.ts +10 -0
  22. package/src/audit-log.ts +122 -0
  23. package/src/auto-gc.ts +98 -29
  24. package/src/auto-recall.ts +33 -4
  25. package/src/capture-engine.ts +372 -23
  26. package/src/cascade-forget.ts +3 -2
  27. package/src/cli.ts +107 -5
  28. package/src/compat.ts +22 -0
  29. package/src/confidence-tracker.ts +210 -47
  30. package/src/conflict-store.ts +12 -3
  31. package/src/consolidation-engine.ts +296 -8
  32. package/src/context-collapse-renderer.ts +222 -0
  33. package/src/context-composer-stable-selection.ts +5 -1
  34. package/src/context-composer.ts +239 -4
  35. package/src/context-reconstructor.ts +404 -0
  36. package/src/conversation-importer.ts +441 -0
  37. package/src/data-checkup.ts +232 -0
  38. package/src/decay-engine.ts +84 -0
  39. package/src/distill-lock.ts +122 -0
  40. package/src/doctor.ts +8 -10
  41. package/src/dream-pipeline.ts +227 -0
  42. package/src/emotion-detector.ts +90 -0
  43. package/src/eval.ts +3 -2
  44. package/src/forget-engine.ts +269 -0
  45. package/src/frequency-tracker.ts +175 -0
  46. package/src/graph-export.ts +644 -0
  47. package/src/ingest.ts +36 -6
  48. package/src/interference-detector.ts +187 -0
  49. package/src/kg-extractor.ts +15 -24
  50. package/src/kg-store.ts +2 -1
  51. package/src/language-hook.ts +67 -0
  52. package/src/llm-client.ts +289 -6
  53. package/src/mcp-server.ts +707 -80
  54. package/src/memory-assets.ts +22 -5
  55. package/src/memory-evolution.ts +365 -0
  56. package/src/memory-lint.ts +429 -0
  57. package/src/memory-output.ts +69 -0
  58. package/src/memory-schema.ts +74 -2
  59. package/src/narrative-schema.ts +79 -0
  60. package/src/narrative-tagger.ts +164 -0
  61. package/src/pii-detector.ts +98 -0
  62. package/src/post-retrieval-filter.ts +76 -0
  63. package/src/prediction-engine.ts +253 -0
  64. package/src/preference-slots.ts +65 -2
  65. package/src/prospective-memory.ts +215 -0
  66. package/src/query-expander.ts +67 -0
  67. package/src/recall-governor.ts +183 -0
  68. package/src/retention-policy.ts +90 -0
  69. package/src/retriever.ts +828 -55
  70. package/src/rif.ts +34 -8
  71. package/src/runtime-config.ts +23 -8
  72. package/src/scope-inventory.ts +4 -3
  73. package/src/scope-policy.ts +4 -0
  74. package/src/session-distiller.ts +307 -0
  75. package/src/session-output.ts +40 -0
  76. package/src/session-schema.ts +37 -0
  77. package/src/session-store.ts +7 -2
  78. package/src/skill-engine.ts +208 -0
  79. package/src/skill-promotion.ts +293 -0
  80. package/src/skill-schema.ts +39 -0
  81. package/src/store.ts +96 -10
  82. package/src/topic-tag.ts +66 -0
  83. package/src/tracker.ts +3 -1
  84. package/src/ui-server.ts +80 -8
  85. package/src/workflow-observation-scope-review.ts +2 -1
  86. package/src/workflow-observation-store.ts +2 -1
  87. package/src/write-verifier.ts +126 -0
  88. package/.claude-plugin/CLAUDE.md +0 -29
  89. package/.claude-plugin/marketplace.json +0 -18
  90. package/.github/workflows/ci.yml +0 -28
  91. package/.mcp.json +0 -9
  92. package/CHANGELOG.md +0 -51
  93. package/DEMO.md +0 -94
  94. package/EVAL.md +0 -89
  95. package/FAILURES.md +0 -75
  96. package/OPERATIONS.md +0 -59
  97. package/ROADMAP.md +0 -198
  98. package/assets/recallnest-workbench.png +0 -0
  99. package/assets/ui/app.js +0 -807
  100. package/assets/ui/index.html +0 -154
  101. package/assets/ui/styles.css +0 -605
  102. package/assets/wechat-qr.jpg +0 -0
  103. package/daily-reflections/2026-03-05.md +0 -15
  104. package/daily-reflections/2026-03-06.md +0 -15
  105. package/daily-reflections/2026-03-07.md +0 -7
  106. package/daily-reflections/2026-03-08.md +0 -34
  107. package/daily-reflections/2026-03-09.md +0 -35
  108. package/daily-reflections/2026-03-10.md +0 -36
  109. package/daily-reflections/2026-03-11.md +0 -35
  110. package/daily-reflections/2026-03-12.md +0 -36
  111. package/daily-reflections/2026-03-13.md +0 -35
  112. package/daily-reflections/2026-03-14.md +0 -35
  113. package/daily-reflections/2026-03-15.md +0 -35
  114. package/daily-reflections/2026-03-16.md +0 -41
  115. package/daily-reflections/2026-03-17.md +0 -10
  116. package/daily-reflections/2026-03-18.md +0 -10
  117. package/daily-reflections/2026-03-19.md +0 -10
  118. package/daily-reflections/2026-03-20.md +0 -10
  119. package/daily-reflections/2026-03-21.md +0 -10
  120. package/daily-reflections/2026-03-22.md +0 -10
  121. package/daily-reflections/2026-03-23.md +0 -10
  122. package/daily-reflections/2026-03-24.md +0 -10
  123. package/daily-reflections/2026-03-25.md +0 -32
  124. package/daily-reflections/2026-03-26.md +0 -37
  125. package/daily-reflections/2026-03-27.md +0 -35
  126. package/daily-reflections/2026-03-28.md +0 -35
  127. package/daily-reflections/2026-03-29.md +0 -35
  128. package/daily-reflections/2026-03-30.md +0 -36
  129. package/daily-reflections/2026-03-31.md +0 -35
  130. package/daily-reflections/2026-04-01.md +0 -35
  131. package/docs/api-reference.md +0 -1291
  132. package/docs/architecture.md +0 -168
  133. package/docs/memory-boundary-contract.md +0 -226
  134. package/docs/memory-categories.md +0 -119
  135. package/docs/pipeline-audit-ultramemory.md +0 -177
  136. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  137. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  138. package/docs/plans/mcp-tiering-plan.md +0 -126
  139. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  140. package/docs/self-evolution.md +0 -198
  141. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  142. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  143. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  144. package/eval/continuity/results-2026-03-16.md +0 -117
  145. package/eval/continuity/results-2026-03-19.md +0 -453
  146. package/eval/continuity/runbook.md +0 -110
  147. package/eval/continuity/seed-cases.md +0 -54
  148. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  149. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  150. package/eval/reports/2026-03-06-baseline.md +0 -82
  151. package/eval/reports/query-expansion.md +0 -138
  152. package/lm +0 -19
  153. package/plan-v1.2-distributable.md +0 -105
  154. package/scripts/archive-distilled-junk.ts +0 -116
  155. package/scripts/backfill-l0l1l2.ts +0 -98
  156. package/scripts/blind-scan.ts +0 -588
  157. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  158. package/scripts/cleanup-step2-template-dups.ts +0 -113
  159. package/scripts/cleanup-step3-stale-content.ts +0 -149
  160. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  161. package/scripts/delete-distilled-junk.ts +0 -88
  162. package/scripts/distill-facts.ts +0 -305
  163. package/scripts/health-check.ts +0 -343
  164. package/scripts/incremental-ingest.sh +0 -71
  165. package/scripts/mlp-health-check.ts +0 -382
  166. package/scripts/pull-claude-desktop.ts +0 -515
  167. package/scripts/re-ingest-cc.sh +0 -66
  168. package/scripts/rebalance-memory-health.ts +0 -181
  169. package/scripts/run-desktop-ingest.sh +0 -11
  170. package/scripts/start-server.sh +0 -12
  171. package/scripts/viz-dashboard.html +0 -423
  172. package/scripts/viz-export.ts +0 -703
  173. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  174. package/src/__tests__/auto-consolidation.test.ts +0 -168
  175. package/src/__tests__/auto-recall.test.ts +0 -131
  176. package/src/__tests__/batch-dedup.test.ts +0 -65
  177. package/src/__tests__/capture-engine.test.ts +0 -1142
  178. package/src/__tests__/capture-heuristic.test.ts +0 -171
  179. package/src/__tests__/cascade-forget.test.ts +0 -210
  180. package/src/__tests__/category-min-scores.test.ts +0 -33
  181. package/src/__tests__/confidence-tracker.test.ts +0 -196
  182. package/src/__tests__/conflict-advisor.test.ts +0 -206
  183. package/src/__tests__/conflict-escalation.test.ts +0 -149
  184. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  185. package/src/__tests__/conflict-output.test.ts +0 -85
  186. package/src/__tests__/conflict-store.test.ts +0 -374
  187. package/src/__tests__/consolidation-engine.test.ts +0 -181
  188. package/src/__tests__/context-composer-pins.test.ts +0 -64
  189. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  190. package/src/__tests__/context-renderer.test.ts +0 -91
  191. package/src/__tests__/continuity-eval.test.ts +0 -275
  192. package/src/__tests__/continuity-schema.test.ts +0 -97
  193. package/src/__tests__/core-recall-eval.test.ts +0 -235
  194. package/src/__tests__/core-summary.test.ts +0 -109
  195. package/src/__tests__/dedup-actions.test.ts +0 -58
  196. package/src/__tests__/deterministic-id.test.ts +0 -39
  197. package/src/__tests__/doctor.test.ts +0 -303
  198. package/src/__tests__/drill-down.test.ts +0 -108
  199. package/src/__tests__/embedder.test.ts +0 -122
  200. package/src/__tests__/entity-resolver.test.ts +0 -58
  201. package/src/__tests__/event-segmenter.test.ts +0 -153
  202. package/src/__tests__/hotness-blend.test.ts +0 -76
  203. package/src/__tests__/ingest.test.ts +0 -360
  204. package/src/__tests__/kg-extractor.test.ts +0 -184
  205. package/src/__tests__/kg-store.test.ts +0 -198
  206. package/src/__tests__/llm-consolidation.test.ts +0 -226
  207. package/src/__tests__/memory-boundaries.test.ts +0 -237
  208. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/ppr-traversal.test.ts +0 -256
  212. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  213. package/src/__tests__/preference-matcher.test.ts +0 -230
  214. package/src/__tests__/preference-slots.test.ts +0 -106
  215. package/src/__tests__/prospective-memory.test.ts +0 -283
  216. package/src/__tests__/query-entity-detector.test.ts +0 -171
  217. package/src/__tests__/query-expander.test.ts +0 -34
  218. package/src/__tests__/recall-mode.test.ts +0 -64
  219. package/src/__tests__/result-synthesizer.test.ts +0 -198
  220. package/src/__tests__/resume-context.test.ts +0 -4743
  221. package/src/__tests__/retrieval-trace.test.ts +0 -213
  222. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  223. package/src/__tests__/retroactive-boost.test.ts +0 -223
  224. package/src/__tests__/scope-inventory.test.ts +0 -200
  225. package/src/__tests__/scope-policy.test.ts +0 -69
  226. package/src/__tests__/session-checkpoint.test.ts +0 -259
  227. package/src/__tests__/session-gc.test.ts +0 -215
  228. package/src/__tests__/store-scope-audit.test.ts +0 -81
  229. package/src/__tests__/temporal-parser.test.ts +0 -241
  230. package/src/__tests__/term-registry.test.ts +0 -121
  231. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  232. package/src/__tests__/version-manager.test.ts +0 -213
  233. package/src/__tests__/workflow-observation.test.ts +0 -400
package/src/retriever.ts CHANGED
@@ -8,14 +8,19 @@ import type { Embedder } from "./embedder.js";
8
8
  import { filterNoise } from "./noise-filter.js";
9
9
  import { shouldSkipRetrieval } from "./adaptive-retrieval.js";
10
10
  import { expandQuery } from "./query-expander.js";
11
+ import { detectLang, tokenizeFts } from "./language-hook.js";
11
12
  import { type AccessTracker, computeHotnessScore, parseAccessMetadata } from "./access-tracker.js";
12
- import { weibullDecay, resolveTier } from "./decay-engine.js";
13
+ import { weibullDecay, resolveTier, isDecayExempt, adjustHalfLifeForEmotion, computeArousalBoost } from "./decay-engine.js";
13
14
  import { logWarn } from "./stderr-log.js";
14
15
  import { extractBoundaryMetadata, isDurableMemoryScope, isTranscriptScope } from "./memory-boundaries.js";
15
16
  import type { TraceCollector } from "./retrieval-trace.js";
17
+ import { extractTopicTag } from "./topic-tag.js";
16
18
  import { filterInterference } from "./rif.js";
19
+ import { FrequencyTracker } from "./frequency-tracker.js";
17
20
  import { applyConfidenceWeight } from "./confidence-tracker.js";
18
21
  import { deduplicateByVersionGroup } from "./version-manager.js";
22
+ import { deduplicateByClusterInsight } from "./consolidation-engine.js";
23
+ import { isActiveMemory, recordAccess as recordEvolutionAccess, parseEvolution, computeDecayScore } from "./memory-evolution.js";
19
24
  import {
20
25
  isMultiVectorEnabled,
21
26
  extractMultiVectorText,
@@ -30,6 +35,17 @@ import { isKGModeEnabled } from "./kg-extractor.js";
30
35
  import { detectEntities } from "./query-entity-detector.js";
31
36
  import { buildGraph, pprTraverse } from "./ppr-traversal.js";
32
37
  import { logInfo } from "./stderr-log.js";
38
+ import type { AuditLogger } from "./audit-log.js";
39
+ import { parseEmotion, isEmotionScoringEnabled } from "./memory-schema.js";
40
+ import { parseNarrative, isNarrativeModeEnabled } from "./narrative-schema.js";
41
+ import type { EmotionMetadata } from "./memory-schema.js";
42
+ import { detectEmotion } from "./emotion-detector.js";
43
+ import { shouldReconstruct, reconstruct as runReconstruction } from "./context-reconstructor.js";
44
+ import type {
45
+ ReconstructionLLMClient,
46
+ ReconstructionOutput,
47
+ CandidateExpansionDeps,
48
+ } from "./context-reconstructor.js";
33
49
 
34
50
  // ============================================================================
35
51
  // Types & Configuration
@@ -101,6 +117,31 @@ export interface RetrievalConfig {
101
117
  rifThreshold?: number;
102
118
  /** RIF score ratio: demote if score < ratio * stronger result's score (default: 0.80). */
103
119
  rifScoreRatio?: number;
120
+ /**
121
+ * Source diversity: ensure top-k results span multiple scopes/sessions.
122
+ * When > 0 and candidates come from >= 3 distinct scopes, applies round-robin
123
+ * selection (one per scope first, then fill remaining slots by score).
124
+ * 0 = disabled (default). Recommended: 0.5 (moderate diversity).
125
+ */
126
+ sourceDiversity?: number;
127
+ /**
128
+ * Adaptive candidate pool multiplier for aggregation queries.
129
+ * When a query contains counting/listing signals ("how many", "all the", etc.),
130
+ * candidatePoolSize is multiplied by this factor.
131
+ * 1 = disabled (default). Recommended: 1.5.
132
+ */
133
+ adaptivePoolMultiplier?: number;
134
+ /**
135
+ * Enable multi-hop retrieval: after first-pass results, extract entities
136
+ * and run focused follow-up queries to improve cross-session coverage.
137
+ * Costs 1-3 extra embedding calls per retrieve. Default: false.
138
+ */
139
+ multiHop?: boolean;
140
+ /**
141
+ * Maximum number of entity-focused follow-up queries in multi-hop mode.
142
+ * Default: 3.
143
+ */
144
+ multiHopMaxQueries?: number;
104
145
  }
105
146
 
106
147
  export interface RetrievalContext {
@@ -118,6 +159,16 @@ export interface RetrievalContext {
118
159
  trace?: TraceCollector;
119
160
  /** Enable KG graph traversal (PPR) as an additional retrieval signal. */
120
161
  graph?: boolean;
162
+ /** Override config-level multiHop for this single retrieval call. */
163
+ multiHop?: boolean;
164
+ /** MP-1: Filter results by topicTag stored in metadata. */
165
+ topicTag?: string;
166
+ /** Request constructive retrieval reconstruction */
167
+ reconstruct?: boolean;
168
+ /** F3: Query memories valid at a specific point in time (ms timestamp). */
169
+ validAt?: number;
170
+ /** F3: When true, include expired memories (demoted 80%). Default: false. */
171
+ includeExpired?: boolean;
121
172
  }
122
173
 
123
174
  export interface RetrievalResult extends MemorySearchResult {
@@ -127,9 +178,19 @@ export interface RetrievalResult extends MemorySearchResult {
127
178
  graph?: { score: number; rank: number };
128
179
  fused?: { score: number };
129
180
  reranked?: { score: number };
181
+ /** HP-narrative: true when this result was pulled as a narrative sibling */
182
+ narrativeSibling?: boolean;
130
183
  };
131
184
  }
132
185
 
186
+ /**
187
+ * Phase 4: Extended retrieval result set with optional first-class reconstruction.
188
+ * Backward-compatible — callers treating this as RetrievalResult[] still work.
189
+ */
190
+ export type RetrievalResultSet = RetrievalResult[] & {
191
+ reconstruction?: ReconstructionOutput;
192
+ };
193
+
133
194
  // ============================================================================
134
195
  // Default Configuration
135
196
  // ============================================================================
@@ -180,6 +241,219 @@ function clamp01(value: number, fallback: number): number {
180
241
  return Math.min(1, Math.max(0, value));
181
242
  }
182
243
 
244
+ /**
245
+ * F3: Filter results by temporal validity window.
246
+ * - Expired memories (validUntil < now) are demoted by 80% or excluded.
247
+ * - validAt: only return memories valid at a specific point in time.
248
+ */
249
+ function filterByValidity(
250
+ results: RetrievalResult[],
251
+ now: number,
252
+ options: { validAt?: number; includeExpired?: boolean },
253
+ ): RetrievalResult[] {
254
+ const { validAt, includeExpired } = options;
255
+ const checkTime = validAt ?? now;
256
+
257
+ return results.reduce<RetrievalResult[]>((acc, r) => {
258
+ const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
259
+
260
+ // Point-in-time query: skip memories not yet valid or already expired at checkTime
261
+ if (validAt != null) {
262
+ if (evo.validFrom > checkTime) return acc; // not yet valid
263
+ if (evo.validUntil != null && evo.validUntil < checkTime) {
264
+ if (!includeExpired) return acc;
265
+ // Include but demote 80%
266
+ acc.push({ ...r, score: r.score * 0.2 });
267
+ return acc;
268
+ }
269
+ acc.push(r);
270
+ return acc;
271
+ }
272
+
273
+ // Default: check if expired relative to now
274
+ if (evo.validUntil != null && evo.validUntil < now) {
275
+ if (!includeExpired) return acc;
276
+ acc.push({ ...r, score: r.score * 0.2 });
277
+ return acc;
278
+ }
279
+
280
+ acc.push(r);
281
+ return acc;
282
+ }, []);
283
+ }
284
+
285
+ /** P0.1: Short query detection — ≤ 4 tokens (CJK chars count as 1 token each). */
286
+ const SHORT_QUERY_TOKEN_THRESHOLD = 4;
287
+
288
+ function isShortQuery(query: string): boolean {
289
+ return tokenize(query).length <= SHORT_QUERY_TOKEN_THRESHOLD;
290
+ }
291
+
292
+ /** P0.1: Minimum score discount for short queries to widen the candidate pool. */
293
+ const SHORT_QUERY_MIN_SCORE_FACTOR = 0.6;
294
+
295
+ /** P0.1: Anchor boost factor — how much a perfect anchor match boosts score. */
296
+ const ANCHOR_BOOST_MAX = 0.25;
297
+
298
+ /**
299
+ * P0.1: Boost scores for candidates whose metadata.anchor matches the query.
300
+ * Uses textOverlapScore (token coverage + density) — same as multi-vector blend.
301
+ * Short queries benefit most because anchor text is also short (≤80 chars).
302
+ */
303
+ function applyAnchorBoost(results: RetrievalResult[], query: string): RetrievalResult[] {
304
+ if (results.length === 0) return results;
305
+ return results.map(r => {
306
+ const meta = parseMetadata(r.entry.metadata);
307
+ const anchor = meta.anchor;
308
+ if (typeof anchor !== "string" || !anchor) return r;
309
+ const overlap = textOverlapScore(query, anchor);
310
+ if (overlap <= 0) return r;
311
+ // Boost proportional to overlap quality, capped at ANCHOR_BOOST_MAX
312
+ const boost = 1 + overlap * ANCHOR_BOOST_MAX;
313
+ return { ...r, score: r.score * boost };
314
+ });
315
+ }
316
+
317
+ /**
318
+ * Emotion scoring stage: boost memories matching query emotional tone.
319
+ * No-op when query is neutral or memory lacks emotion data.
320
+ */
321
+ export function applyEmotionWeight(
322
+ results: Array<{ score: number; metadata: string; [k: string]: unknown }>,
323
+ queryEmotion: EmotionMetadata | null,
324
+ ): typeof results {
325
+ if (!queryEmotion || Math.abs(queryEmotion.valence) < 0.2) {
326
+ return results;
327
+ }
328
+ return results.map(r => {
329
+ const memEmotion = parseEmotion(r.metadata);
330
+ if (!memEmotion) return r;
331
+ const alignment = 1 - Math.abs(queryEmotion.valence - memEmotion.valence) / 2;
332
+ const boost = 1.0 + 0.15 * alignment;
333
+ return { ...r, score: r.score * boost };
334
+ });
335
+ }
336
+
337
+ // ============================================================================
338
+ // Source Diversity — round-robin across scopes for multi-session coverage
339
+ // ============================================================================
340
+
341
+ /** Minimum distinct sources to activate diversity logic. */
342
+ const SOURCE_DIVERSITY_MIN_SOURCES = 3;
343
+
344
+ /**
345
+ * Ensures top-k results span multiple scopes/sessions via round-robin.
346
+ * Each distinct source gets its top-1 result first, remaining slots fill by score.
347
+ */
348
+ function applySourceDiversity(results: RetrievalResult[], limit: number): RetrievalResult[] {
349
+ if (results.length <= limit) return results;
350
+
351
+ // Group by scope
352
+ const byScope = new Map<string, RetrievalResult[]>();
353
+ for (const r of results) {
354
+ const key = r.entry.scope || "__default__";
355
+ const bucket = byScope.get(key);
356
+ if (bucket) bucket.push(r);
357
+ else byScope.set(key, [r]);
358
+ }
359
+
360
+ if (byScope.size < SOURCE_DIVERSITY_MIN_SOURCES) {
361
+ return results.slice(0, limit);
362
+ }
363
+
364
+ // Round-robin: each scope contributes its top-1
365
+ const selected: RetrievalResult[] = [];
366
+ const usedIds = new Set<string>();
367
+
368
+ // Sort scopes by their best result score (highest first)
369
+ const scopeEntries = [...byScope.entries()].sort(
370
+ (a, b) => (b[1][0]?.score ?? 0) - (a[1][0]?.score ?? 0),
371
+ );
372
+
373
+ for (const [, items] of scopeEntries) {
374
+ if (selected.length >= limit) break;
375
+ const top = items[0];
376
+ if (top && !usedIds.has(top.entry.id)) {
377
+ selected.push(top);
378
+ usedIds.add(top.entry.id);
379
+ }
380
+ }
381
+
382
+ // Fill remaining slots by global score
383
+ for (const r of results) {
384
+ if (selected.length >= limit) break;
385
+ if (!usedIds.has(r.entry.id)) {
386
+ selected.push(r);
387
+ usedIds.add(r.entry.id);
388
+ }
389
+ }
390
+
391
+ return selected;
392
+ }
393
+
394
+ // ============================================================================
395
+ // Adaptive Candidate Pool — detect aggregation queries
396
+ // ============================================================================
397
+
398
+ const AGGREGATION_EN_RE = /\b(?:how many|how much|all the|every|list all|total number|count)\b/i;
399
+ const AGGREGATION_CN_RE = /(?:多少[个件条只]?|一共|总共|所有的?|列举|有几[个件条只])/u;
400
+
401
+ function isAggregationQuery(query: string): boolean {
402
+ return AGGREGATION_EN_RE.test(query) || AGGREGATION_CN_RE.test(query);
403
+ }
404
+
405
+ // ============================================================================
406
+ // Multi-hop Entity Extraction — extract entities from retrieval results
407
+ // ============================================================================
408
+
409
+ const MULTI_HOP_STOP_WORDS = new Set([
410
+ "the", "a", "an", "is", "are", "was", "were", "be", "been",
411
+ "have", "has", "had", "will", "would", "can", "could", "should",
412
+ "do", "does", "did", "not", "no", "but", "and", "or", "if",
413
+ "this", "that", "these", "those", "it", "its", "user", "assistant",
414
+ "my", "your", "his", "her", "our", "their", "i", "you", "he", "she",
415
+ "we", "they", "me", "him", "us", "them", "who", "what", "where",
416
+ "when", "how", "why", "which", "about", "with", "from", "for",
417
+ "into", "also", "just", "very", "some", "more", "most", "other",
418
+ ]);
419
+
420
+ const COMMON_CJK_STOP = /^(用户|助手|可以|需要|已经|没有|什么|这个|那个|因为|所以|但是|如果)$/;
421
+
422
+ /**
423
+ * Extract salient entities from retrieval result texts.
424
+ * Focuses on proper nouns and CJK named entities.
425
+ * Returns unique entities sorted by frequency (most common first).
426
+ */
427
+ function extractEntitiesFromResults(results: RetrievalResult[]): string[] {
428
+ const freq = new Map<string, number>();
429
+
430
+ for (const r of results) {
431
+ const text = r.entry.text;
432
+
433
+ // Capitalized multi-word entities: "Adobe Premiere Pro", "Sony A7III"
434
+ const capPattern = /\b([A-Z][a-zA-Z0-9]*(?:[\s\-][A-Z][a-zA-Z0-9]*)*)\b/g;
435
+ let match: RegExpExecArray | null;
436
+ while ((match = capPattern.exec(text)) !== null) {
437
+ const entity = match[1].trim();
438
+ if (entity.length < 2 || MULTI_HOP_STOP_WORDS.has(entity.toLowerCase())) continue;
439
+ freq.set(entity, (freq.get(entity) ?? 0) + 1);
440
+ }
441
+
442
+ // CJK named entities: sequences of 2-8 CJK chars (simple heuristic)
443
+ const cjkPattern = /([\p{Script=Han}]{2,8})/gu;
444
+ while ((match = cjkPattern.exec(text)) !== null) {
445
+ const entity = match[1];
446
+ if (COMMON_CJK_STOP.test(entity)) continue;
447
+ freq.set(entity, (freq.get(entity) ?? 0) + 1);
448
+ }
449
+ }
450
+
451
+ // Sort by frequency descending, then by length descending (prefer specific entities)
452
+ return [...freq.entries()]
453
+ .sort((a, b) => b[1] - a[1] || b[0].length - a[0].length)
454
+ .map(([entity]) => entity);
455
+ }
456
+
183
457
  function parseMetadata(raw?: string): Record<string, unknown> {
184
458
  try {
185
459
  return JSON.parse(raw || "{}") as Record<string, unknown>;
@@ -353,6 +627,9 @@ function cosineSimilarity(a: number[], b: number[]): number {
353
627
  export class MemoryRetriever {
354
628
  private accessTracker?: AccessTracker;
355
629
  private kgStore?: KGStore;
630
+ private frequencyTracker?: FrequencyTracker;
631
+ private auditLogger?: AuditLogger;
632
+ private llmClient?: ReconstructionLLMClient;
356
633
  /**
357
634
  * Session-scoped suppression list (dopamine-inspired "do not disturb").
358
635
  * IDs in this set get a score penalty during retrieval but are NOT deleted.
@@ -376,6 +653,21 @@ export class MemoryRetriever {
376
653
  this.kgStore = kgStore;
377
654
  }
378
655
 
656
+ /** P0.2: Attach a FrequencyTracker for hit-count based boosting. */
657
+ setFrequencyTracker(tracker: FrequencyTracker): void {
658
+ this.frequencyTracker = tracker;
659
+ }
660
+
661
+ /** F-1: Attach an AuditLogger for recording retrieve operations. */
662
+ setAuditLogger(logger: AuditLogger): void {
663
+ this.auditLogger = logger;
664
+ }
665
+
666
+ /** Attach an LLM client for constructive retrieval reconstruction. */
667
+ setLLMClient(client: ReconstructionLLMClient): void {
668
+ this.llmClient = client;
669
+ }
670
+
379
671
  /** Suppress a memory ID for the current session (score × 0.3). */
380
672
  suppressForSession(id: string): void {
381
673
  this.sessionSuppressed.add(id);
@@ -386,13 +678,15 @@ export class MemoryRetriever {
386
678
  this.sessionSuppressed.clear();
387
679
  }
388
680
 
389
- /** Resolve the minimum score for a given category, falling back to hardMinScore. */
390
- private minScoreFor(category: string): number {
681
+ /** Resolve the minimum score for a given category, falling back to hardMinScore.
682
+ * P0.1: When shortQuery is true, thresholds are lowered to widen the candidate pool. */
683
+ private minScoreFor(category: string, shortQuery = false): number {
391
684
  const map = this.config.categoryMinScores ?? DEFAULT_CATEGORY_MIN_SCORES;
392
- return map[category] ?? this.config.hardMinScore;
685
+ const base = map[category] ?? this.config.hardMinScore;
686
+ return shortQuery ? base * SHORT_QUERY_MIN_SCORE_FACTOR : base;
393
687
  }
394
688
 
395
- async retrieve(context: RetrievalContext): Promise<RetrievalResult[]> {
689
+ async retrieve(context: RetrievalContext): Promise<RetrievalResultSet> {
396
690
  const { query, limit, scopeFilter, category, includeArchived, trace, graph } = context;
397
691
  const safeLimit = clampInt(limit, 1, 20);
398
692
 
@@ -411,18 +705,152 @@ export class MemoryRetriever {
411
705
  results = await this.hybridRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace, graph);
412
706
  }
413
707
 
708
+ // LME-2: Multi-hop retrieval — extract entities from first-pass results,
709
+ // run focused follow-up queries, merge to improve cross-session coverage.
710
+ const useMultiHop = context.multiHop ?? this.config.multiHop ?? false;
711
+ if (useMultiHop && results.length > 0) {
712
+ results = await this.multiHopExpand(results, context);
713
+ }
714
+
715
+ // MP-1: Topic Tag post-filter — only keep results matching the requested topicTag
716
+ if (context.topicTag && results.length > 0) {
717
+ const tag = context.topicTag.toLowerCase();
718
+ results = results.filter(r => {
719
+ const entryTag = extractTopicTag(r.entry.metadata);
720
+ return entryTag?.toLowerCase() === tag;
721
+ });
722
+ }
723
+
724
+ // F3: Temporal validity filter — expired memories excluded by default
725
+ if (results.length > 0) {
726
+ results = filterByValidity(results, Date.now(), {
727
+ validAt: context.validAt,
728
+ includeExpired: context.includeExpired,
729
+ });
730
+ }
731
+
732
+ // P0.2: Record frequency hits for returned results (manual queries only)
733
+ // Moved after topicTag filter so filtered-out entries are not reinforced.
734
+ if (this.frequencyTracker && results.length > 0 && context.source !== "auto-recall") {
735
+ this.frequencyTracker.recordHits(results.map(r => r.entry.id));
736
+ }
737
+
738
+ // A-3: Record evolution access counts (async, non-blocking)
739
+ // Moved after topicTag filter so filtered-out entries are not reinforced.
740
+ if (results.length > 0 && context.source !== "auto-recall" && this.store.update) {
741
+ Promise.resolve().then(async () => {
742
+ for (const r of results) {
743
+ try {
744
+ const updated = recordEvolutionAccess(r.entry.metadata);
745
+ await this.store.update!(r.entry.id, { metadata: updated });
746
+ } catch (err) { console.error("[recallnest] Evolution access tracking failed:", err instanceof Error ? err.message : String(err)); }
747
+ }
748
+ });
749
+ }
750
+
414
751
  // Record access for returned results (async, non-blocking).
415
752
  // Only reinforce on manual retrieval — auto-recall must not strengthen noise.
416
- // Tier 3.2: pass similarity scores for novelty gating — only novel retrievals
417
- // (high query-result distance) trigger access reinforcement.
753
+ // Tier 3.2: pass raw vector similarity scores for novelty gating — only novel
754
+ // retrievals (high query-result distance) trigger access reinforcement.
418
755
  if (this.accessTracker && results.length > 0 && context.source !== "auto-recall") {
419
756
  this.accessTracker.recordAccess(
420
757
  results.map(r => r.entry.id),
421
- results.map(r => r.score),
758
+ results.map(r => r.sources.vector?.score),
422
759
  );
423
760
  }
424
761
 
425
- return results;
762
+ // F-1: Audit log — record retrieve operation (non-blocking, silent on failure)
763
+ try {
764
+ this.auditLogger?.log({
765
+ operation: "retrieve",
766
+ scope: context.scopeFilter?.[0],
767
+ actor: context.source || "manual",
768
+ details: `query="${context.query.slice(0, 80)}" hits=${results.length}`,
769
+ });
770
+ } catch {
771
+ // Audit must never block retrieval
772
+ }
773
+
774
+ // Phase 4: Constructive retrieval — first-class reconstruction (no metadata hack)
775
+ const resultSet = results as RetrievalResultSet;
776
+ const constructiveFlag = process.env.RECALLNEST_CONSTRUCTIVE_RETRIEVAL === "true";
777
+ if (shouldReconstruct({
778
+ flagEnabled: constructiveFlag,
779
+ callerOptIn: context.reconstruct === true,
780
+ resultCount: results.length,
781
+ llmAvailable: this.llmClient
782
+ ? ("isAvailable" in this.llmClient ? (this.llmClient as { isAvailable(): boolean }).isAvailable() : true)
783
+ : false,
784
+ })) {
785
+ try {
786
+ const expansionDeps = this.buildExpansionDeps(context);
787
+ const reconstruction = await runReconstruction(
788
+ { query: context.query, results, mode: "search" },
789
+ this.llmClient!,
790
+ expansionDeps,
791
+ );
792
+ resultSet.reconstruction = reconstruction;
793
+ } catch {
794
+ // Silent degradation — return raw results
795
+ }
796
+ }
797
+
798
+ return resultSet;
799
+ }
800
+
801
+ /**
802
+ * LME-2: Multi-hop expansion — extract entities from first-pass results,
803
+ * run entity-focused follow-up queries, merge with original results.
804
+ * Improves cross-session coverage for aggregation/counting queries.
805
+ */
806
+ private async multiHopExpand(
807
+ firstPass: RetrievalResult[],
808
+ context: RetrievalContext,
809
+ ): Promise<RetrievalResult[]> {
810
+ const maxQueries = this.config.multiHopMaxQueries ?? 3;
811
+ const entities = extractEntitiesFromResults(firstPass);
812
+ if (entities.length === 0) return firstPass;
813
+
814
+ // Pick top entities that aren't already well-covered in the query
815
+ const queryLower = context.query.toLowerCase();
816
+ const novelEntities = entities
817
+ .filter(e => !queryLower.includes(e.toLowerCase()))
818
+ .slice(0, maxQueries);
819
+
820
+ if (novelEntities.length === 0) return firstPass;
821
+
822
+ // Run follow-up queries in parallel (entity + original query context)
823
+ const followUpPromises = novelEntities.map(entity =>
824
+ this.hybridRetrieval(
825
+ `${entity} ${context.query}`,
826
+ context.limit,
827
+ context.scopeFilter,
828
+ context.category,
829
+ context.includeArchived,
830
+ undefined, // no trace for follow-up
831
+ context.graph,
832
+ ).catch(() => [] as RetrievalResult[]),
833
+ );
834
+ const followUpResults = await Promise.all(followUpPromises);
835
+
836
+ // Merge: deduplicate by memory ID, keep highest score
837
+ const merged = new Map<string, RetrievalResult>();
838
+ for (const r of firstPass) {
839
+ merged.set(r.entry.id, r);
840
+ }
841
+ for (const batch of followUpResults) {
842
+ for (const r of batch) {
843
+ const existing = merged.get(r.entry.id);
844
+ if (!existing || r.score > existing.score) {
845
+ merged.set(r.entry.id, r);
846
+ }
847
+ }
848
+ }
849
+
850
+ // Sort by score descending, limit
851
+ return [...merged.values()]
852
+ .sort((a, b) => b.score - a.score)
853
+ .slice(0, context.limit);
426
854
  }
427
855
 
428
856
  private async vectorOnlyRetrieval(
@@ -439,7 +867,12 @@ export class MemoryRetriever {
439
867
 
440
868
  trace?.startStage("vector_search", 0);
441
869
  const queryVector = await this.embedder.embedQuery(searchQuery);
442
- const results = await this.store.vectorSearch(queryVector, limit, this.config.minScore, scopeFilter);
870
+ // P0.1: Lower minScore for short queries
871
+ const shortQ = isShortQuery(searchQuery);
872
+ const vectorMinScore = shortQ
873
+ ? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
874
+ : this.config.minScore;
875
+ const results = await this.store.vectorSearch(queryVector, limit, vectorMinScore, scopeFilter);
443
876
  trace?.endStage(results.length, results.map(r => r.score));
444
877
 
445
878
  // Filter by category if specified
@@ -464,8 +897,11 @@ export class MemoryRetriever {
464
897
  },
465
898
  } as RetrievalResult));
466
899
 
467
- trace?.startStage("recency_boost", mapped.length);
468
- const boosted = this.applyRecencyBoost(mapped);
900
+ // P0.1: Anchor boost for vector-only path
901
+ const anchorBoosted = applyAnchorBoost(mapped, searchQuery);
902
+
903
+ trace?.startStage("recency_boost", anchorBoosted.length);
904
+ const boosted = this.applyRecencyBoost(anchorBoosted);
469
905
  trace?.endStage(boosted.length, boosted.map(r => r.score));
470
906
 
471
907
  trace?.startStage("importance_weight", boosted.length);
@@ -476,8 +912,16 @@ export class MemoryRetriever {
476
912
  const confidenceWeighted = this.applyConfidence(weighted);
477
913
  trace?.endStage(confidenceWeighted.length, confidenceWeighted.map(r => r.score));
478
914
 
479
- trace?.startStage("boundary_weight", confidenceWeighted.length);
480
- const boundaryWeighted = this.applyBoundaryWeight(confidenceWeighted);
915
+ let afterEmotionVector = confidenceWeighted;
916
+ if (isEmotionScoringEnabled()) {
917
+ const queryEmotion = detectEmotion(query);
918
+ const adapted = confidenceWeighted.map(r => ({ ...r, metadata: r.entry.metadata ?? "{}" }));
919
+ const emotioned = applyEmotionWeight(adapted, queryEmotion);
920
+ afterEmotionVector = confidenceWeighted.map((r, i) => ({ ...r, score: emotioned[i]?.score ?? r.score }));
921
+ }
922
+
923
+ trace?.startStage("boundary_weight", afterEmotionVector.length);
924
+ const boundaryWeighted = this.applyBoundaryWeight(afterEmotionVector);
481
925
  trace?.endStage(boundaryWeighted.length, boundaryWeighted.map(r => r.score));
482
926
 
483
927
  trace?.startStage("asset_type_weight", boundaryWeighted.length);
@@ -492,22 +936,39 @@ export class MemoryRetriever {
492
936
  const timeDecayed = this.applyTimeDecay(lengthNormalized);
493
937
  trace?.endStage(timeDecayed.length, timeDecayed.map(r => r.score));
494
938
 
939
+ // B-1/E-1: Evolution decay blend — boost memories with high composite decay score
940
+ const evolutionBlended = this.applyEvolutionDecayBlend(timeDecayed);
941
+
942
+ // E-1: Access count boost — memories retrieved more often get a score nudge
943
+ const accessBoosted = this.applyAccessCountBoost(evolutionBlended);
944
+
495
945
  // Hotness blend: boost frequently + recently accessed memories
496
- trace?.startStage("hotness_blend", timeDecayed.length);
497
- const hotnessBlended = this.applyHotnessBlend(timeDecayed);
946
+ trace?.startStage("hotness_blend", accessBoosted.length);
947
+ const hotnessBlended = this.applyHotnessBlend(accessBoosted);
498
948
  trace?.endStage(hotnessBlended.length, hotnessBlended.map(r => r.score));
499
949
 
950
+ // P0.2: Frequency boost — repeatedly retrieved memories score higher
951
+ trace?.startStage("frequency_boost", hotnessBlended.length);
952
+ const frequencyBoosted = this.applyFrequencyBoost(hotnessBlended);
953
+ trace?.endStage(frequencyBoosted.length, frequencyBoosted.map(r => r.score));
954
+
500
955
  // Session suppression: temporarily penalize "do not disturb" memories
501
- const afterSuppression = this.applySessionSuppression(hotnessBlended);
956
+ const afterSuppression = this.applySessionSuppression(frequencyBoosted);
957
+ const rescored = [...afterSuppression].sort((a, b) => b.score - a.score);
502
958
 
503
- trace?.startStage("hard_min_score", afterSuppression.length);
504
- const hardFiltered = afterSuppression.filter(r => r.score >= this.minScoreFor(r.entry.category));
959
+ trace?.startStage("hard_min_score", rescored.length);
960
+ const hardFiltered = rescored.filter(r => r.score >= this.minScoreFor(r.entry.category, shortQ));
505
961
  trace?.endStage(hardFiltered.length, hardFiltered.map(r => r.score));
506
962
 
507
- trace?.startStage("noise_filter", hardFiltered.length);
963
+ // Evolution status filter: exclude superseded/archived/consolidated memories
964
+ const evolutionFiltered = includeArchived
965
+ ? hardFiltered
966
+ : hardFiltered.filter(r => isActiveMemory(r.entry.metadata));
967
+
968
+ trace?.startStage("noise_filter", evolutionFiltered.length);
508
969
  const denoised = this.config.filterNoise
509
- ? filterNoise(hardFiltered, r => r.entry.text)
510
- : hardFiltered;
970
+ ? filterNoise(evolutionFiltered, r => r.entry.text)
971
+ : evolutionFiltered;
511
972
  trace?.endStage(denoised.length, denoised.map(r => r.score));
512
973
 
513
974
  // RIF: demote near-duplicate weak results to improve diversity
@@ -523,8 +984,16 @@ export class MemoryRetriever {
523
984
  // Tier 3.3: Version group dedup — keep only the top-ranked version per group
524
985
  const versionDeduped = deduplicateByVersionGroup(deduplicated);
525
986
 
526
- trace?.startStage("final_limit", versionDeduped.length);
527
- const final = versionDeduped.slice(0, limit);
987
+ // LC-P2: Cluster insight dedup — prefer cluster summary over individual source memories
988
+ const clusterDeduped = deduplicateByClusterInsight(versionDeduped);
989
+
990
+ // Source diversity: round-robin across scopes for multi-session coverage
991
+ const diversified = (this.config.sourceDiversity ?? 0) > 0
992
+ ? applySourceDiversity(clusterDeduped, limit)
993
+ : clusterDeduped;
994
+
995
+ trace?.startStage("final_limit", diversified.length);
996
+ const final = diversified.slice(0, limit);
528
997
  trace?.endStage(final.length, final.map(r => r.score));
529
998
 
530
999
  return final;
@@ -539,7 +1008,14 @@ export class MemoryRetriever {
539
1008
  trace?: TraceCollector,
540
1009
  graph?: boolean,
541
1010
  ): Promise<RetrievalResult[]> {
542
- const candidatePoolSize = Math.max(this.config.candidatePoolSize, limit * 2);
1011
+ // Adaptive pool: widen candidate pool for aggregation queries ("how many", "all the")
1012
+ const multiplier = (this.config.adaptivePoolMultiplier ?? 1) > 1 && isAggregationQuery(query)
1013
+ ? (this.config.adaptivePoolMultiplier ?? 1)
1014
+ : 1;
1015
+ const candidatePoolSize = Math.max(
1016
+ Math.ceil(this.config.candidatePoolSize * multiplier),
1017
+ limit * 2,
1018
+ );
543
1019
 
544
1020
  // Temporal reasoning: extract time constraint and clean query for semantic search
545
1021
  const temporal = parseTemporalQuery(query);
@@ -554,10 +1030,14 @@ export class MemoryRetriever {
554
1030
  ? this.runPPRSearch(query, scopeFilter?.[0], trace)
555
1031
  : Promise.resolve([] as RetrievalResult[]);
556
1032
 
1033
+ // Pre-tokenize expanded query for BM25 so CJK text produces meaningful FTS matches
1034
+ const expandedQuery = expandQuery(searchQuery);
1035
+ const ftsQuery = tokenizeFts(expandedQuery, detectLang(expandedQuery));
1036
+
557
1037
  trace?.startStage("vector_search", 0);
558
1038
  const [vectorResults, bm25Results, pprResults] = await Promise.all([
559
1039
  this.runVectorSearch(queryVector, candidatePoolSize, scopeFilter, category, includeArchived),
560
- this.runBM25Search(expandQuery(searchQuery), candidatePoolSize, scopeFilter, category, includeArchived),
1040
+ this.runBM25Search(ftsQuery, candidatePoolSize, scopeFilter, category, includeArchived),
561
1041
  pprPromise,
562
1042
  ]);
563
1043
  trace?.endStage(vectorResults.length, vectorResults.map(r => r.score));
@@ -565,9 +1045,10 @@ export class MemoryRetriever {
565
1045
  trace?.startStage("bm25_search", 0);
566
1046
  trace?.endStage(bm25Results.length, bm25Results.map(r => r.score));
567
1047
 
568
- // Fuse results using RRF (async: validates BM25-only entries exist in store)
1048
+ // Fuse results using weighted score fusion (async: validates BM25-only entries exist in store)
1049
+ const shortQ = isShortQuery(searchQuery);
569
1050
  trace?.startStage("rrf_fusion", vectorResults.length + bm25Results.length + pprResults.length);
570
- const fusedResults = await this.fuseResults(vectorResults, bm25Results, pprResults);
1051
+ const fusedResults = await this.fuseResults(vectorResults, bm25Results, pprResults, shortQ);
571
1052
  trace?.endStage(fusedResults.length, fusedResults.map(r => r.score));
572
1053
 
573
1054
  // Temporal filter: when query has a time constraint, remove non-matching candidates
@@ -582,9 +1063,18 @@ export class MemoryRetriever {
582
1063
  const multiVecBlended = this.applyMultiVectorBlend(temporalFiltered, searchQuery);
583
1064
  trace?.endStage(multiVecBlended.length, multiVecBlended.map(r => r.score));
584
1065
 
1066
+ // P0.1: Anchor boost — short query terms matching short anchor text
1067
+ trace?.startStage("anchor_boost", multiVecBlended.length);
1068
+ const anchorBoosted = applyAnchorBoost(multiVecBlended, searchQuery);
1069
+ trace?.endStage(anchorBoosted.length, anchorBoosted.map(r => r.score));
1070
+
585
1071
  // Apply minimum score threshold
586
- trace?.startStage("min_score_filter", multiVecBlended.length);
587
- const filtered = multiVecBlended.filter(r => r.score >= this.config.minScore);
1072
+ trace?.startStage("min_score_filter", anchorBoosted.length);
1073
+ // P0.1: Lower minScore for short queries to widen the candidate pool
1074
+ const effectiveMinScore = isShortQuery(searchQuery)
1075
+ ? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
1076
+ : this.config.minScore;
1077
+ const filtered = anchorBoosted.filter(r => r.score >= effectiveMinScore);
588
1078
  trace?.endStage(filtered.length, filtered.map(r => r.score));
589
1079
 
590
1080
  // Rerank if enabled
@@ -609,9 +1099,18 @@ export class MemoryRetriever {
609
1099
  const confidenceWeighted = this.applyConfidence(importanceWeighted);
610
1100
  trace?.endStage(confidenceWeighted.length, confidenceWeighted.map(r => r.score));
611
1101
 
1102
+ // Emotion scoring: boost memories matching query emotional tone
1103
+ let afterEmotion = confidenceWeighted;
1104
+ if (isEmotionScoringEnabled()) {
1105
+ const queryEmotion = detectEmotion(query);
1106
+ const adapted = confidenceWeighted.map(r => ({ ...r, metadata: r.entry.metadata ?? "{}" }));
1107
+ const emotioned = applyEmotionWeight(adapted, queryEmotion);
1108
+ afterEmotion = confidenceWeighted.map((r, i) => ({ ...r, score: emotioned[i]?.score ?? r.score }));
1109
+ }
1110
+
612
1111
  // Prefer higher-authority durable memories over raw evidence when scores are close.
613
- trace?.startStage("boundary_weight", confidenceWeighted.length);
614
- const boundaryWeighted = this.applyBoundaryWeight(confidenceWeighted);
1112
+ trace?.startStage("boundary_weight", afterEmotion.length);
1113
+ const boundaryWeighted = this.applyBoundaryWeight(afterEmotion);
615
1114
  trace?.endStage(boundaryWeighted.length, boundaryWeighted.map(r => r.score));
616
1115
 
617
1116
  // Separate pinned assets from synthesized briefs.
@@ -629,21 +1128,40 @@ export class MemoryRetriever {
629
1128
  const timeDecayed = this.applyTimeDecay(lengthNormalized);
630
1129
  trace?.endStage(timeDecayed.length, timeDecayed.map(r => r.score));
631
1130
 
1131
+ // B-1/E-1: Evolution decay blend — boost memories with high composite decay score
1132
+ const evolutionBlended = this.applyEvolutionDecayBlend(timeDecayed);
1133
+
1134
+ // E-1: Access count boost — memories retrieved more often get a score nudge
1135
+ const accessBoosted = this.applyAccessCountBoost(evolutionBlended);
1136
+
632
1137
  // Hotness blend: boost frequently + recently accessed memories
633
- trace?.startStage("hotness_blend", timeDecayed.length);
634
- const hotnessBlended = this.applyHotnessBlend(timeDecayed);
1138
+ trace?.startStage("hotness_blend", accessBoosted.length);
1139
+ const hotnessBlended = this.applyHotnessBlend(accessBoosted);
635
1140
  trace?.endStage(hotnessBlended.length, hotnessBlended.map(r => r.score));
636
1141
 
1142
+ // P0.2: Frequency boost — repeatedly retrieved memories score higher
1143
+ trace?.startStage("frequency_boost", hotnessBlended.length);
1144
+ const frequencyBoosted = this.applyFrequencyBoost(hotnessBlended);
1145
+ trace?.endStage(frequencyBoosted.length, frequencyBoosted.map(r => r.score));
1146
+
637
1147
  // Hard minimum score cutoff (post all scoring stages)
638
- trace?.startStage("hard_min_score", hotnessBlended.length);
639
- const hardFiltered = hotnessBlended.filter(r => r.score >= this.minScoreFor(r.entry.category));
1148
+ // P0.1: lower thresholds for short queries (shortQ declared earlier for fuseResults)
1149
+ const rescored = [...frequencyBoosted].sort((a, b) => b.score - a.score);
1150
+
1151
+ trace?.startStage("hard_min_score", rescored.length);
1152
+ const hardFiltered = rescored.filter(r => r.score >= this.minScoreFor(r.entry.category, shortQ));
640
1153
  trace?.endStage(hardFiltered.length, hardFiltered.map(r => r.score));
641
1154
 
1155
+ // Evolution status filter: exclude superseded/archived/consolidated memories
1156
+ const evolutionFiltered = includeArchived
1157
+ ? hardFiltered
1158
+ : hardFiltered.filter(r => isActiveMemory(r.entry.metadata));
1159
+
642
1160
  // Filter noise
643
- trace?.startStage("noise_filter", hardFiltered.length);
1161
+ trace?.startStage("noise_filter", evolutionFiltered.length);
644
1162
  const denoised = this.config.filterNoise
645
- ? filterNoise(hardFiltered, r => r.entry.text)
646
- : hardFiltered;
1163
+ ? filterNoise(evolutionFiltered, r => r.entry.text)
1164
+ : evolutionFiltered;
647
1165
  trace?.endStage(denoised.length, denoised.map(r => r.score));
648
1166
 
649
1167
  // RIF: demote near-duplicate weak results to improve diversity
@@ -659,8 +1177,16 @@ export class MemoryRetriever {
659
1177
  // Tier 3.3: Version group dedup — keep only the top-ranked version per group
660
1178
  const versionDeduped = deduplicateByVersionGroup(deduplicated);
661
1179
 
662
- trace?.startStage("final_limit", versionDeduped.length);
663
- const final = versionDeduped.slice(0, limit);
1180
+ // LC-P2: Cluster insight dedup — prefer cluster summary over individual source memories
1181
+ const clusterDeduped = deduplicateByClusterInsight(versionDeduped);
1182
+
1183
+ // Source diversity: round-robin across scopes for multi-session coverage
1184
+ const diversified = (this.config.sourceDiversity ?? 0) > 0
1185
+ ? applySourceDiversity(clusterDeduped, limit)
1186
+ : clusterDeduped;
1187
+
1188
+ trace?.startStage("final_limit", diversified.length);
1189
+ const final = diversified.slice(0, limit);
664
1190
  trace?.endStage(final.length, final.map(r => r.score));
665
1191
 
666
1192
  return final;
@@ -806,7 +1332,18 @@ export class MemoryRetriever {
806
1332
  vectorResults: Array<MemorySearchResult & { rank: number }>,
807
1333
  bm25Results: Array<MemorySearchResult & { rank: number }>,
808
1334
  graphResults: RetrievalResult[] = [],
1335
+ shortQuery: boolean = false,
809
1336
  ): Promise<RetrievalResult[]> {
1337
+ // Adaptive weights: short queries (≤4 tokens) favor BM25 for exact keyword matching.
1338
+ // For normal queries, use config weights directly.
1339
+ const vW = shortQuery
1340
+ ? this.config.vectorWeight * 0.7
1341
+ : this.config.vectorWeight;
1342
+ const bW = shortQuery
1343
+ ? Math.min(this.config.bm25Weight * 1.5, 0.6)
1344
+ : this.config.bm25Weight;
1345
+ const totalW = vW + bW;
1346
+
810
1347
  // Create maps for quick lookup
811
1348
  const vectorMap = new Map<string, MemorySearchResult & { rank: number }>();
812
1349
  const bm25Map = new Map<string, MemorySearchResult & { rank: number }>();
@@ -827,7 +1364,7 @@ export class MemoryRetriever {
827
1364
  // Get all unique document IDs
828
1365
  const allIds = new Set([...vectorMap.keys(), ...bm25Map.keys(), ...graphMap.keys()]);
829
1366
 
830
- // Calculate RRF scores
1367
+ // Calculate fused scores using configurable weights
831
1368
  const fusedResults: RetrievalResult[] = [];
832
1369
 
833
1370
  for (const id of allIds) {
@@ -850,26 +1387,29 @@ export class MemoryRetriever {
850
1387
  // Use the result with more complete data (prefer vector > graph > BM25)
851
1388
  const baseResult = vectorResult || graphResult || bm25Result!;
852
1389
 
853
- // Use vector similarity as the base score.
854
- // BM25 hit acts as a bonus (keyword match confirms relevance).
855
- // Graph hit acts as a bonus (entity relationship confirms relevance).
856
1390
  const vectorScore = vectorResult ? vectorResult.score : 0;
857
- const bm25Hit = bm25Result ? 1 : 0;
1391
+ const bm25Score = bm25Result ? bm25Result.score : 0;
858
1392
  const graphScore = graphResult?.sources.graph?.score ?? 0;
859
1393
 
860
- // Base = vector score; BM25 hit boosts by up to 15%; graph boosts by up to 20%
861
- // Graph-only results use their PPR score so entity-connected memories surface.
862
1394
  let fusedScore: number;
863
- if (vectorResult) {
1395
+ if (vectorResult && bm25Result) {
1396
+ // Both signals agree: weighted average + 5% dual-match confirmation bonus.
1397
+ // Graph hit adds up to 15% on top.
864
1398
  fusedScore = clamp01(
865
- vectorScore + (bm25Hit * 0.15 * vectorScore) + (graphScore * 0.20),
1399
+ ((vW * vectorScore + bW * bm25Score) / totalW) * 1.05 + (graphScore * 0.15),
866
1400
  0.1,
867
1401
  );
1402
+ } else if (vectorResult) {
1403
+ // Vector only: semantic match, graph as bonus
1404
+ fusedScore = clamp01(vectorScore + (graphScore * 0.15), 0.1);
868
1405
  } else if (graphResult) {
869
- // Graph-only: use graph score as base, BM25 boosts
870
- fusedScore = clamp01(graphScore + (bm25Hit * 0.10), 0.1);
1406
+ // Graph only: entity-relationship match, BM25 as bonus
1407
+ fusedScore = clamp01(graphScore + (bm25Score > 0 ? 0.10 : 0), 0.1);
871
1408
  } else {
872
- fusedScore = clamp01(bm25Result!.score, 0.1);
1409
+ // BM25 only: exact keyword match without semantic confirmation.
1410
+ // Short queries get a small boost since BM25 is more reliable for exact terms.
1411
+ const bm25OnlyBoost = shortQuery ? 1.15 : 1.0;
1412
+ fusedScore = clamp01(bm25Score * bm25OnlyBoost, 0.1);
873
1413
  }
874
1414
 
875
1415
  fusedResults.push({
@@ -1178,22 +1718,33 @@ export class MemoryRetriever {
1178
1718
 
1179
1719
  const now = Date.now();
1180
1720
  const decayed = results.map(r => {
1721
+ // HP-7: Exempt core/pinned/recently-accessed from decay
1722
+ if (isDecayExempt(r.entry.metadata, r.entry.importance)) return r;
1723
+
1181
1724
  const ts = (r.entry.timestamp && r.entry.timestamp > 0) ? r.entry.timestamp : now;
1182
1725
  const ageDays = (now - ts) / 86_400_000;
1183
1726
 
1184
1727
  // Use access-reinforced half-life if tracker is attached
1185
- const halfLife = this.accessTracker
1728
+ let halfLife = this.accessTracker
1186
1729
  ? this.accessTracker.computeEffectiveHalfLife(baseHalfLife, r.entry.metadata)
1187
1730
  : baseHalfLife;
1188
1731
 
1732
+ // HP-emo: Emotion-adjusted half-life — strong emotion slows forgetting
1733
+ const emotion = isEmotionScoringEnabled() ? parseEmotion(r.entry.metadata) : null;
1734
+ if (emotion) {
1735
+ halfLife = adjustHalfLifeForEmotion(halfLife, emotion);
1736
+ }
1737
+
1189
1738
  // Resolve tier from metadata (defaults to peripheral)
1190
1739
  const tier = resolveTier(r.entry.metadata);
1191
1740
 
1192
1741
  // Weibull decay with tier-specific shape and floor
1193
1742
  const factor = weibullDecay(ageDays, halfLife, tier);
1743
+ // HP-emo: Arousal boost — flashbulb memory effect (1.0–1.1x)
1744
+ const arousalFactor = emotion ? computeArousalBoost(emotion) : 1.0;
1194
1745
  return {
1195
1746
  ...r,
1196
- score: clamp01(r.score * factor, r.score * 0.3),
1747
+ score: clamp01(r.score * arousalFactor * factor, r.score * 0.3),
1197
1748
  };
1198
1749
  });
1199
1750
 
@@ -1208,6 +1759,214 @@ export class MemoryRetriever {
1208
1759
  *
1209
1760
  * No-op when hotnessWeight is 0 or AccessTracker is absent.
1210
1761
  */
1762
+ /**
1763
+ * B-1/E-1: Evolution decay blend — factor in composite decay score
1764
+ * (time × frequency × importance) as a scoring signal.
1765
+ * Weight increased from 0.05 to 0.10 with B-3 LLM importance assessment.
1766
+ */
1767
+ private applyEvolutionDecayBlend(results: RetrievalResult[]): RetrievalResult[] {
1768
+ const EVOLUTION_BLEND_WEIGHT = 0.10;
1769
+ return results.map(r => {
1770
+ // HP-7: Exempt entries get max decay score (1.0) → no penalty
1771
+ if (isDecayExempt(r.entry.metadata, r.entry.importance)) {
1772
+ return {
1773
+ ...r,
1774
+ score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + 1.0 * EVOLUTION_BLEND_WEIGHT, 0),
1775
+ };
1776
+ }
1777
+ const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
1778
+ const decay = computeDecayScore(evo, r.entry.importance, undefined, r.entry.metadata);
1779
+ return {
1780
+ ...r,
1781
+ score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + decay * EVOLUTION_BLEND_WEIGHT, 0),
1782
+ };
1783
+ });
1784
+ }
1785
+
1786
+ /**
1787
+ * E-1: Access count boost — memories with higher access_count get a score nudge.
1788
+ * Complements frequency-tracker (query→memory mapping) by operating at
1789
+ * the individual memory level (total retrieval hits across all queries).
1790
+ * Formula: score *= 1 + log2(1 + accessCount) × 0.03, capped at ×1.15.
1791
+ */
1792
+ private applyAccessCountBoost(results: RetrievalResult[]): RetrievalResult[] {
1793
+ return results.map(r => {
1794
+ const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
1795
+ if (evo.accessCount <= 0) return r;
1796
+ const boost = Math.min(0.15, Math.log2(1 + evo.accessCount) * 0.03);
1797
+ return {
1798
+ ...r,
1799
+ score: clamp01(r.score * (1 + boost), 0),
1800
+ };
1801
+ });
1802
+ }
1803
+
1804
+ /**
1805
+ * Phase 4: Build candidate expansion deps from internal stores.
1806
+ * Each dep is only provided if the corresponding store/feature is available.
1807
+ */
1808
+ private buildExpansionDeps(context: RetrievalContext): CandidateExpansionDeps {
1809
+ const deps: CandidateExpansionDeps = {};
1810
+ const scope = context.scopeFilter?.[0];
1811
+
1812
+ // KG neighbor expansion — requires KG mode + attached KG store
1813
+ if (isKGModeEnabled() && this.kgStore) {
1814
+ const kgStore = this.kgStore;
1815
+ const store = this.store;
1816
+ deps.expandViaKG = async (results) => {
1817
+ // Extract unique source_memory_ids from KG triples linked to result entities
1818
+ const entityNames: string[] = [];
1819
+ for (const r of results.slice(0, 5)) {
1820
+ try {
1821
+ const meta = JSON.parse(r.entry.metadata || "{}");
1822
+ if (meta.kg_entities && Array.isArray(meta.kg_entities)) {
1823
+ entityNames.push(...meta.kg_entities);
1824
+ }
1825
+ } catch (err) { console.error("[recallnest] KG entity extraction failed:", err instanceof Error ? err.message : String(err)); }
1826
+ }
1827
+ if (entityNames.length === 0) return [];
1828
+ const unique = [...new Set(entityNames)].slice(0, 5);
1829
+ const neighborhood = await kgStore.getNeighborhood(unique, 1, scope);
1830
+ const sourceIds = new Set<string>();
1831
+ const existingIds = new Set(results.map(r => r.entry.id));
1832
+ for (const n of neighborhood) {
1833
+ for (const t of n.triples) {
1834
+ if (t.source_memory_id && !existingIds.has(t.source_memory_id)) {
1835
+ sourceIds.add(t.source_memory_id);
1836
+ }
1837
+ }
1838
+ }
1839
+ // Fetch entries by ID
1840
+ const expanded: RetrievalResult[] = [];
1841
+ for (const id of [...sourceIds].slice(0, 5)) {
1842
+ const entry = await store.getById(id);
1843
+ if (entry) {
1844
+ expanded.push({ entry, score: 0.5, sources: {} });
1845
+ }
1846
+ }
1847
+ return expanded;
1848
+ };
1849
+ }
1850
+
1851
+ // Evolution chain expansion — walk supersedes/supersededBy links
1852
+ if (this.store.getById) {
1853
+ const store = this.store;
1854
+ deps.expandViaEvolution = async (results) => {
1855
+ const expanded: RetrievalResult[] = [];
1856
+ const existingIds = new Set(results.map(r => r.entry.id));
1857
+ for (const r of results.slice(0, 5)) {
1858
+ const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
1859
+ for (const linkedId of [evo.supersedes, evo.supersededBy].filter(Boolean) as string[]) {
1860
+ if (existingIds.has(linkedId)) continue;
1861
+ const entry = await store.getById(linkedId);
1862
+ if (entry) {
1863
+ existingIds.add(linkedId);
1864
+ expanded.push({ entry, score: r.score * 0.6, sources: {} });
1865
+ }
1866
+ }
1867
+ }
1868
+ return expanded;
1869
+ };
1870
+ }
1871
+
1872
+ // Cluster member expansion — get sourceMemories from cluster insights
1873
+ if (this.store.getById) {
1874
+ const store = this.store;
1875
+ deps.expandViaClusters = async (results) => {
1876
+ const expanded: RetrievalResult[] = [];
1877
+ const existingIds = new Set(results.map(r => r.entry.id));
1878
+ for (const r of results.slice(0, 5)) {
1879
+ try {
1880
+ const meta = JSON.parse(r.entry.metadata || "{}");
1881
+ if ((meta.cluster_insight || meta.cross_memory_pattern) && meta.evolution?.sourceMemories) {
1882
+ for (const memberId of (meta.evolution.sourceMemories as string[]).slice(0, 3)) {
1883
+ if (existingIds.has(memberId)) continue;
1884
+ const entry = await store.getById(memberId);
1885
+ if (entry) {
1886
+ existingIds.add(memberId);
1887
+ expanded.push({ entry, score: r.score * 0.5, sources: {} });
1888
+ }
1889
+ }
1890
+ }
1891
+ } catch (err) { console.error("[recallnest] Cluster member expansion failed:", err instanceof Error ? err.message : String(err)); }
1892
+ }
1893
+ return expanded;
1894
+ };
1895
+ }
1896
+
1897
+ // Narrative sibling expansion — reuse existing expandNarrativeSiblings logic
1898
+ if (isNarrativeModeEnabled()) {
1899
+ deps.expandViaNarrative = async (results) => {
1900
+ const expanded = await this.expandNarrativeSiblings(results);
1901
+ // Return only the new siblings (not in original results)
1902
+ const existingIds = new Set(results.map(r => r.entry.id));
1903
+ return expanded.filter(r => !existingIds.has(r.entry.id));
1904
+ };
1905
+ }
1906
+
1907
+ return deps;
1908
+ }
1909
+
1910
+ /**
1911
+ * HP-narrative: Expand retrieval results with narrative siblings.
1912
+ * When a retrieved memory has narrative metadata, pull other memories
1913
+ * sharing the same generalEventId to provide temporal context.
1914
+ *
1915
+ * Siblings are added with a dampened score (70% of the triggering result)
1916
+ * and capped at 3 siblings per general event to avoid flooding.
1917
+ */
1918
+ async expandNarrativeSiblings(results: RetrievalResult[]): Promise<RetrievalResult[]> {
1919
+ if (!isNarrativeModeEnabled() || results.length === 0) return results;
1920
+
1921
+ // Collect unique generalEventIds from results
1922
+ const eventIdToScore = new Map<string, number>();
1923
+ const existingIds = new Set(results.map(r => r.entry.id));
1924
+
1925
+ for (const r of results) {
1926
+ const narrative = parseNarrative(r.entry.metadata);
1927
+ if (!narrative) continue;
1928
+ const existing = eventIdToScore.get(narrative.generalEventId);
1929
+ if (!existing || r.score > existing) {
1930
+ eventIdToScore.set(narrative.generalEventId, r.score);
1931
+ }
1932
+ }
1933
+
1934
+ if (eventIdToScore.size === 0) return results;
1935
+
1936
+ // Search store for siblings — use scope-based listing and filter by generalEventId
1937
+ const siblings: RetrievalResult[] = [];
1938
+ const MAX_SIBLINGS_PER_EVENT = 3;
1939
+
1940
+ try {
1941
+ const allEntries = await this.store.list(undefined, undefined, 500, 0);
1942
+ for (const entry of allEntries) {
1943
+ if (existingIds.has(entry.id)) continue;
1944
+ const narrative = parseNarrative(entry.metadata);
1945
+ if (!narrative) continue;
1946
+ const triggerScore = eventIdToScore.get(narrative.generalEventId);
1947
+ if (triggerScore === undefined) continue;
1948
+
1949
+ // Count how many siblings we've already added for this event
1950
+ const eventSiblingCount = siblings.filter(s => {
1951
+ const sn = parseNarrative(s.entry.metadata);
1952
+ return sn?.generalEventId === narrative.generalEventId;
1953
+ }).length;
1954
+ if (eventSiblingCount >= MAX_SIBLINGS_PER_EVENT) continue;
1955
+
1956
+ siblings.push({
1957
+ entry,
1958
+ score: triggerScore * 0.7,
1959
+ sources: { narrativeSibling: true },
1960
+ });
1961
+ existingIds.add(entry.id);
1962
+ }
1963
+ } catch {
1964
+ // Non-blocking — if store listing fails, return original results
1965
+ }
1966
+
1967
+ return [...results, ...siblings].sort((a, b) => b.score - a.score);
1968
+ }
1969
+
1211
1970
  private applyHotnessBlend(results: RetrievalResult[]): RetrievalResult[] {
1212
1971
  const raw = this.config.hotnessWeight;
1213
1972
  const alpha = Math.min(1, Math.max(0, Number.isFinite(raw) ? raw : 0));
@@ -1228,6 +1987,20 @@ export class MemoryRetriever {
1228
1987
  return blended.sort((a, b) => b.score - a.score);
1229
1988
  }
1230
1989
 
1990
+ /**
1991
+ * P0.2: Frequency boost — memories that get repeatedly retrieved score higher.
1992
+ * Unlike hotness (access-tracker based, decay-oriented), this is pure hit-count
1993
+ * boost designed to surface "always relevant" memories like daily patrol rules.
1994
+ */
1995
+ private applyFrequencyBoost(results: RetrievalResult[]): RetrievalResult[] {
1996
+ if (!this.frequencyTracker || results.length === 0) return results;
1997
+ return results.map(r => {
1998
+ const multiplier = this.frequencyTracker!.getBoostMultiplier(r.entry.id);
1999
+ if (multiplier <= 1.0) return r;
2000
+ return { ...r, score: Math.min(1, r.score * multiplier) };
2001
+ });
2002
+ }
2003
+
1231
2004
  /**
1232
2005
  * MMR-inspired diversity filter: greedily select results that are both
1233
2006
  * relevant (high score) and diverse (low similarity to already-selected).