recallnest 1.4.1 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +4 -0
- package/CLAUDE.md +7 -3
- package/README.md +202 -130
- package/README_CN.md +263 -190
- package/bin/recallnest.mjs +62 -0
- package/config.json +55 -0
- package/daily-recall.sh +23 -0
- package/integrations/setup-all.sh +31 -0
- package/logs/daily-recall.error.log +0 -0
- package/logs/daily-recall.log +30 -0
- package/logs/ingest-2026-04-11.log +110 -0
- package/logs/ingest-2026-04-12.log +90 -0
- package/logs/launchagent-stderr.log +0 -0
- package/logs/launchagent-stdout.log +0 -0
- package/package.json +10 -3
- package/src/access-tracker.ts +3 -2
- package/src/activity-counter.ts +109 -0
- package/src/adaptive-retrieval.ts +6 -5
- package/src/admission-control.ts +163 -0
- package/src/anchor-generator.ts +80 -0
- package/src/api-server.ts +10 -0
- package/src/audit-log.ts +122 -0
- package/src/auto-gc.ts +98 -29
- package/src/auto-recall.ts +33 -4
- package/src/capture-engine.ts +372 -23
- package/src/cascade-forget.ts +3 -2
- package/src/cli.ts +107 -5
- package/src/compat.ts +22 -0
- package/src/confidence-tracker.ts +210 -47
- package/src/conflict-store.ts +12 -3
- package/src/consolidation-engine.ts +296 -8
- package/src/context-collapse-renderer.ts +222 -0
- package/src/context-composer-stable-selection.ts +5 -1
- package/src/context-composer.ts +239 -4
- package/src/context-reconstructor.ts +404 -0
- package/src/conversation-importer.ts +441 -0
- package/src/data-checkup.ts +232 -0
- package/src/decay-engine.ts +84 -0
- package/src/distill-lock.ts +122 -0
- package/src/doctor.ts +8 -10
- package/src/dream-pipeline.ts +227 -0
- package/src/emotion-detector.ts +90 -0
- package/src/eval.ts +3 -2
- package/src/forget-engine.ts +269 -0
- package/src/frequency-tracker.ts +175 -0
- package/src/graph-export.ts +644 -0
- package/src/ingest.ts +36 -6
- package/src/interference-detector.ts +187 -0
- package/src/kg-extractor.ts +15 -24
- package/src/kg-store.ts +2 -1
- package/src/language-hook.ts +67 -0
- package/src/llm-client.ts +289 -6
- package/src/mcp-server.ts +707 -80
- package/src/memory-assets.ts +22 -5
- package/src/memory-evolution.ts +365 -0
- package/src/memory-lint.ts +429 -0
- package/src/memory-output.ts +69 -0
- package/src/memory-schema.ts +74 -2
- package/src/narrative-schema.ts +79 -0
- package/src/narrative-tagger.ts +164 -0
- package/src/pii-detector.ts +98 -0
- package/src/post-retrieval-filter.ts +76 -0
- package/src/prediction-engine.ts +253 -0
- package/src/preference-slots.ts +65 -2
- package/src/prospective-memory.ts +215 -0
- package/src/query-expander.ts +67 -0
- package/src/recall-governor.ts +183 -0
- package/src/retention-policy.ts +90 -0
- package/src/retriever.ts +828 -55
- package/src/rif.ts +34 -8
- package/src/runtime-config.ts +23 -8
- package/src/scope-inventory.ts +4 -3
- package/src/scope-policy.ts +4 -0
- package/src/session-distiller.ts +307 -0
- package/src/session-output.ts +40 -0
- package/src/session-schema.ts +37 -0
- package/src/session-store.ts +7 -2
- package/src/skill-engine.ts +208 -0
- package/src/skill-promotion.ts +293 -0
- package/src/skill-schema.ts +39 -0
- package/src/store.ts +96 -10
- package/src/topic-tag.ts +66 -0
- package/src/tracker.ts +3 -1
- package/src/ui-server.ts +80 -8
- package/src/workflow-observation-scope-review.ts +2 -1
- package/src/workflow-observation-store.ts +2 -1
- package/src/write-verifier.ts +126 -0
- package/.claude-plugin/CLAUDE.md +0 -29
- package/.claude-plugin/marketplace.json +0 -18
- package/.github/workflows/ci.yml +0 -28
- package/.mcp.json +0 -9
- package/CHANGELOG.md +0 -51
- package/DEMO.md +0 -94
- package/EVAL.md +0 -89
- package/FAILURES.md +0 -75
- package/OPERATIONS.md +0 -59
- package/ROADMAP.md +0 -198
- package/assets/recallnest-workbench.png +0 -0
- package/assets/ui/app.js +0 -807
- package/assets/ui/index.html +0 -154
- package/assets/ui/styles.css +0 -605
- package/assets/wechat-qr.jpg +0 -0
- package/daily-reflections/2026-03-05.md +0 -15
- package/daily-reflections/2026-03-06.md +0 -15
- package/daily-reflections/2026-03-07.md +0 -7
- package/daily-reflections/2026-03-08.md +0 -34
- package/daily-reflections/2026-03-09.md +0 -35
- package/daily-reflections/2026-03-10.md +0 -36
- package/daily-reflections/2026-03-11.md +0 -35
- package/daily-reflections/2026-03-12.md +0 -36
- package/daily-reflections/2026-03-13.md +0 -35
- package/daily-reflections/2026-03-14.md +0 -35
- package/daily-reflections/2026-03-15.md +0 -35
- package/daily-reflections/2026-03-16.md +0 -41
- package/daily-reflections/2026-03-17.md +0 -10
- package/daily-reflections/2026-03-18.md +0 -10
- package/daily-reflections/2026-03-19.md +0 -10
- package/daily-reflections/2026-03-20.md +0 -10
- package/daily-reflections/2026-03-21.md +0 -10
- package/daily-reflections/2026-03-22.md +0 -10
- package/daily-reflections/2026-03-23.md +0 -10
- package/daily-reflections/2026-03-24.md +0 -10
- package/daily-reflections/2026-03-25.md +0 -32
- package/daily-reflections/2026-03-26.md +0 -37
- package/daily-reflections/2026-03-27.md +0 -35
- package/daily-reflections/2026-03-28.md +0 -35
- package/daily-reflections/2026-03-29.md +0 -35
- package/daily-reflections/2026-03-30.md +0 -36
- package/daily-reflections/2026-03-31.md +0 -35
- package/daily-reflections/2026-04-01.md +0 -35
- package/docs/api-reference.md +0 -1291
- package/docs/architecture.md +0 -168
- package/docs/memory-boundary-contract.md +0 -226
- package/docs/memory-categories.md +0 -119
- package/docs/pipeline-audit-ultramemory.md +0 -177
- package/docs/plans/2026-03-08-query-expansion.md +0 -270
- package/docs/plans/2026-03-16-continuity-layer.md +0 -196
- package/docs/plans/mcp-tiering-plan.md +0 -126
- package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
- package/docs/self-evolution.md +0 -198
- package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
- package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
- package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
- package/eval/continuity/results-2026-03-16.md +0 -117
- package/eval/continuity/results-2026-03-19.md +0 -453
- package/eval/continuity/runbook.md +0 -110
- package/eval/continuity/seed-cases.md +0 -54
- package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
- package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
- package/eval/reports/2026-03-06-baseline.md +0 -82
- package/eval/reports/query-expansion.md +0 -138
- package/lm +0 -19
- package/plan-v1.2-distributable.md +0 -105
- package/scripts/archive-distilled-junk.ts +0 -116
- package/scripts/backfill-l0l1l2.ts +0 -98
- package/scripts/blind-scan.ts +0 -588
- package/scripts/cleanup-step1-exact-dups.ts +0 -117
- package/scripts/cleanup-step2-template-dups.ts +0 -113
- package/scripts/cleanup-step3-stale-content.ts +0 -149
- package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
- package/scripts/delete-distilled-junk.ts +0 -88
- package/scripts/distill-facts.ts +0 -305
- package/scripts/health-check.ts +0 -343
- package/scripts/incremental-ingest.sh +0 -71
- package/scripts/mlp-health-check.ts +0 -382
- package/scripts/pull-claude-desktop.ts +0 -515
- package/scripts/re-ingest-cc.sh +0 -66
- package/scripts/rebalance-memory-health.ts +0 -181
- package/scripts/run-desktop-ingest.sh +0 -11
- package/scripts/start-server.sh +0 -12
- package/scripts/viz-dashboard.html +0 -423
- package/scripts/viz-export.ts +0 -703
- package/src/__tests__/access-tracker-novelty.test.ts +0 -228
- package/src/__tests__/auto-consolidation.test.ts +0 -168
- package/src/__tests__/auto-recall.test.ts +0 -131
- package/src/__tests__/batch-dedup.test.ts +0 -65
- package/src/__tests__/capture-engine.test.ts +0 -1142
- package/src/__tests__/capture-heuristic.test.ts +0 -171
- package/src/__tests__/cascade-forget.test.ts +0 -210
- package/src/__tests__/category-min-scores.test.ts +0 -33
- package/src/__tests__/confidence-tracker.test.ts +0 -196
- package/src/__tests__/conflict-advisor.test.ts +0 -206
- package/src/__tests__/conflict-escalation.test.ts +0 -149
- package/src/__tests__/conflict-lifecycle.test.ts +0 -83
- package/src/__tests__/conflict-output.test.ts +0 -85
- package/src/__tests__/conflict-store.test.ts +0 -374
- package/src/__tests__/consolidation-engine.test.ts +0 -181
- package/src/__tests__/context-composer-pins.test.ts +0 -64
- package/src/__tests__/context-composer-task-selection.test.ts +0 -24
- package/src/__tests__/context-renderer.test.ts +0 -91
- package/src/__tests__/continuity-eval.test.ts +0 -275
- package/src/__tests__/continuity-schema.test.ts +0 -97
- package/src/__tests__/core-recall-eval.test.ts +0 -235
- package/src/__tests__/core-summary.test.ts +0 -109
- package/src/__tests__/dedup-actions.test.ts +0 -58
- package/src/__tests__/deterministic-id.test.ts +0 -39
- package/src/__tests__/doctor.test.ts +0 -303
- package/src/__tests__/drill-down.test.ts +0 -108
- package/src/__tests__/embedder.test.ts +0 -122
- package/src/__tests__/entity-resolver.test.ts +0 -58
- package/src/__tests__/event-segmenter.test.ts +0 -153
- package/src/__tests__/hotness-blend.test.ts +0 -76
- package/src/__tests__/ingest.test.ts +0 -360
- package/src/__tests__/kg-extractor.test.ts +0 -184
- package/src/__tests__/kg-store.test.ts +0 -198
- package/src/__tests__/llm-consolidation.test.ts +0 -226
- package/src/__tests__/memory-boundaries.test.ts +0 -237
- package/src/__tests__/memory-health-rebalance.test.ts +0 -144
- package/src/__tests__/memory-output.test.ts +0 -215
- package/src/__tests__/multi-vector.test.ts +0 -242
- package/src/__tests__/ppr-traversal.test.ts +0 -256
- package/src/__tests__/preference-guard-smoke.test.ts +0 -211
- package/src/__tests__/preference-matcher.test.ts +0 -230
- package/src/__tests__/preference-slots.test.ts +0 -106
- package/src/__tests__/prospective-memory.test.ts +0 -283
- package/src/__tests__/query-entity-detector.test.ts +0 -171
- package/src/__tests__/query-expander.test.ts +0 -34
- package/src/__tests__/recall-mode.test.ts +0 -64
- package/src/__tests__/result-synthesizer.test.ts +0 -198
- package/src/__tests__/resume-context.test.ts +0 -4743
- package/src/__tests__/retrieval-trace.test.ts +0 -213
- package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
- package/src/__tests__/retroactive-boost.test.ts +0 -223
- package/src/__tests__/scope-inventory.test.ts +0 -200
- package/src/__tests__/scope-policy.test.ts +0 -69
- package/src/__tests__/session-checkpoint.test.ts +0 -259
- package/src/__tests__/session-gc.test.ts +0 -215
- package/src/__tests__/store-scope-audit.test.ts +0 -81
- package/src/__tests__/temporal-parser.test.ts +0 -241
- package/src/__tests__/term-registry.test.ts +0 -121
- package/src/__tests__/tool-output-compressor.test.ts +0 -133
- package/src/__tests__/version-manager.test.ts +0 -213
- package/src/__tests__/workflow-observation.test.ts +0 -400
package/src/retriever.ts
CHANGED
|
@@ -8,14 +8,19 @@ import type { Embedder } from "./embedder.js";
|
|
|
8
8
|
import { filterNoise } from "./noise-filter.js";
|
|
9
9
|
import { shouldSkipRetrieval } from "./adaptive-retrieval.js";
|
|
10
10
|
import { expandQuery } from "./query-expander.js";
|
|
11
|
+
import { detectLang, tokenizeFts } from "./language-hook.js";
|
|
11
12
|
import { type AccessTracker, computeHotnessScore, parseAccessMetadata } from "./access-tracker.js";
|
|
12
|
-
import { weibullDecay, resolveTier } from "./decay-engine.js";
|
|
13
|
+
import { weibullDecay, resolveTier, isDecayExempt, adjustHalfLifeForEmotion, computeArousalBoost } from "./decay-engine.js";
|
|
13
14
|
import { logWarn } from "./stderr-log.js";
|
|
14
15
|
import { extractBoundaryMetadata, isDurableMemoryScope, isTranscriptScope } from "./memory-boundaries.js";
|
|
15
16
|
import type { TraceCollector } from "./retrieval-trace.js";
|
|
17
|
+
import { extractTopicTag } from "./topic-tag.js";
|
|
16
18
|
import { filterInterference } from "./rif.js";
|
|
19
|
+
import { FrequencyTracker } from "./frequency-tracker.js";
|
|
17
20
|
import { applyConfidenceWeight } from "./confidence-tracker.js";
|
|
18
21
|
import { deduplicateByVersionGroup } from "./version-manager.js";
|
|
22
|
+
import { deduplicateByClusterInsight } from "./consolidation-engine.js";
|
|
23
|
+
import { isActiveMemory, recordAccess as recordEvolutionAccess, parseEvolution, computeDecayScore } from "./memory-evolution.js";
|
|
19
24
|
import {
|
|
20
25
|
isMultiVectorEnabled,
|
|
21
26
|
extractMultiVectorText,
|
|
@@ -30,6 +35,17 @@ import { isKGModeEnabled } from "./kg-extractor.js";
|
|
|
30
35
|
import { detectEntities } from "./query-entity-detector.js";
|
|
31
36
|
import { buildGraph, pprTraverse } from "./ppr-traversal.js";
|
|
32
37
|
import { logInfo } from "./stderr-log.js";
|
|
38
|
+
import type { AuditLogger } from "./audit-log.js";
|
|
39
|
+
import { parseEmotion, isEmotionScoringEnabled } from "./memory-schema.js";
|
|
40
|
+
import { parseNarrative, isNarrativeModeEnabled } from "./narrative-schema.js";
|
|
41
|
+
import type { EmotionMetadata } from "./memory-schema.js";
|
|
42
|
+
import { detectEmotion } from "./emotion-detector.js";
|
|
43
|
+
import { shouldReconstruct, reconstruct as runReconstruction } from "./context-reconstructor.js";
|
|
44
|
+
import type {
|
|
45
|
+
ReconstructionLLMClient,
|
|
46
|
+
ReconstructionOutput,
|
|
47
|
+
CandidateExpansionDeps,
|
|
48
|
+
} from "./context-reconstructor.js";
|
|
33
49
|
|
|
34
50
|
// ============================================================================
|
|
35
51
|
// Types & Configuration
|
|
@@ -101,6 +117,31 @@ export interface RetrievalConfig {
|
|
|
101
117
|
rifThreshold?: number;
|
|
102
118
|
/** RIF score ratio: demote if score < ratio * stronger result's score (default: 0.80). */
|
|
103
119
|
rifScoreRatio?: number;
|
|
120
|
+
/**
|
|
121
|
+
* Source diversity: ensure top-k results span multiple scopes/sessions.
|
|
122
|
+
* When > 0 and candidates come from >= 3 distinct scopes, applies round-robin
|
|
123
|
+
* selection (one per scope first, then fill remaining slots by score).
|
|
124
|
+
* 0 = disabled (default). Recommended: 0.5 (moderate diversity).
|
|
125
|
+
*/
|
|
126
|
+
sourceDiversity?: number;
|
|
127
|
+
/**
|
|
128
|
+
* Adaptive candidate pool multiplier for aggregation queries.
|
|
129
|
+
* When a query contains counting/listing signals ("how many", "all the", etc.),
|
|
130
|
+
* candidatePoolSize is multiplied by this factor.
|
|
131
|
+
* 1 = disabled (default). Recommended: 1.5.
|
|
132
|
+
*/
|
|
133
|
+
adaptivePoolMultiplier?: number;
|
|
134
|
+
/**
|
|
135
|
+
* Enable multi-hop retrieval: after first-pass results, extract entities
|
|
136
|
+
* and run focused follow-up queries to improve cross-session coverage.
|
|
137
|
+
* Costs 1-3 extra embedding calls per retrieve. Default: false.
|
|
138
|
+
*/
|
|
139
|
+
multiHop?: boolean;
|
|
140
|
+
/**
|
|
141
|
+
* Maximum number of entity-focused follow-up queries in multi-hop mode.
|
|
142
|
+
* Default: 3.
|
|
143
|
+
*/
|
|
144
|
+
multiHopMaxQueries?: number;
|
|
104
145
|
}
|
|
105
146
|
|
|
106
147
|
export interface RetrievalContext {
|
|
@@ -118,6 +159,16 @@ export interface RetrievalContext {
|
|
|
118
159
|
trace?: TraceCollector;
|
|
119
160
|
/** Enable KG graph traversal (PPR) as an additional retrieval signal. */
|
|
120
161
|
graph?: boolean;
|
|
162
|
+
/** Override config-level multiHop for this single retrieval call. */
|
|
163
|
+
multiHop?: boolean;
|
|
164
|
+
/** MP-1: Filter results by topicTag stored in metadata. */
|
|
165
|
+
topicTag?: string;
|
|
166
|
+
/** Request constructive retrieval reconstruction */
|
|
167
|
+
reconstruct?: boolean;
|
|
168
|
+
/** F3: Query memories valid at a specific point in time (ms timestamp). */
|
|
169
|
+
validAt?: number;
|
|
170
|
+
/** F3: When true, include expired memories (demoted 80%). Default: false. */
|
|
171
|
+
includeExpired?: boolean;
|
|
121
172
|
}
|
|
122
173
|
|
|
123
174
|
export interface RetrievalResult extends MemorySearchResult {
|
|
@@ -127,9 +178,19 @@ export interface RetrievalResult extends MemorySearchResult {
|
|
|
127
178
|
graph?: { score: number; rank: number };
|
|
128
179
|
fused?: { score: number };
|
|
129
180
|
reranked?: { score: number };
|
|
181
|
+
/** HP-narrative: true when this result was pulled as a narrative sibling */
|
|
182
|
+
narrativeSibling?: boolean;
|
|
130
183
|
};
|
|
131
184
|
}
|
|
132
185
|
|
|
186
|
+
/**
|
|
187
|
+
* Phase 4: Extended retrieval result set with optional first-class reconstruction.
|
|
188
|
+
* Backward-compatible — callers treating this as RetrievalResult[] still work.
|
|
189
|
+
*/
|
|
190
|
+
export type RetrievalResultSet = RetrievalResult[] & {
|
|
191
|
+
reconstruction?: ReconstructionOutput;
|
|
192
|
+
};
|
|
193
|
+
|
|
133
194
|
// ============================================================================
|
|
134
195
|
// Default Configuration
|
|
135
196
|
// ============================================================================
|
|
@@ -180,6 +241,219 @@ function clamp01(value: number, fallback: number): number {
|
|
|
180
241
|
return Math.min(1, Math.max(0, value));
|
|
181
242
|
}
|
|
182
243
|
|
|
244
|
+
/**
|
|
245
|
+
* F3: Filter results by temporal validity window.
|
|
246
|
+
* - Expired memories (validUntil < now) are demoted by 80% or excluded.
|
|
247
|
+
* - validAt: only return memories valid at a specific point in time.
|
|
248
|
+
*/
|
|
249
|
+
function filterByValidity(
|
|
250
|
+
results: RetrievalResult[],
|
|
251
|
+
now: number,
|
|
252
|
+
options: { validAt?: number; includeExpired?: boolean },
|
|
253
|
+
): RetrievalResult[] {
|
|
254
|
+
const { validAt, includeExpired } = options;
|
|
255
|
+
const checkTime = validAt ?? now;
|
|
256
|
+
|
|
257
|
+
return results.reduce<RetrievalResult[]>((acc, r) => {
|
|
258
|
+
const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
|
|
259
|
+
|
|
260
|
+
// Point-in-time query: skip memories not yet valid or already expired at checkTime
|
|
261
|
+
if (validAt != null) {
|
|
262
|
+
if (evo.validFrom > checkTime) return acc; // not yet valid
|
|
263
|
+
if (evo.validUntil != null && evo.validUntil < checkTime) {
|
|
264
|
+
if (!includeExpired) return acc;
|
|
265
|
+
// Include but demote 80%
|
|
266
|
+
acc.push({ ...r, score: r.score * 0.2 });
|
|
267
|
+
return acc;
|
|
268
|
+
}
|
|
269
|
+
acc.push(r);
|
|
270
|
+
return acc;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Default: check if expired relative to now
|
|
274
|
+
if (evo.validUntil != null && evo.validUntil < now) {
|
|
275
|
+
if (!includeExpired) return acc;
|
|
276
|
+
acc.push({ ...r, score: r.score * 0.2 });
|
|
277
|
+
return acc;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
acc.push(r);
|
|
281
|
+
return acc;
|
|
282
|
+
}, []);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** P0.1: Short query detection — ≤ 4 tokens (CJK chars count as 1 token each). */
|
|
286
|
+
const SHORT_QUERY_TOKEN_THRESHOLD = 4;
|
|
287
|
+
|
|
288
|
+
function isShortQuery(query: string): boolean {
|
|
289
|
+
return tokenize(query).length <= SHORT_QUERY_TOKEN_THRESHOLD;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** P0.1: Minimum score discount for short queries to widen the candidate pool. */
|
|
293
|
+
const SHORT_QUERY_MIN_SCORE_FACTOR = 0.6;
|
|
294
|
+
|
|
295
|
+
/** P0.1: Anchor boost factor — how much a perfect anchor match boosts score. */
|
|
296
|
+
const ANCHOR_BOOST_MAX = 0.25;
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* P0.1: Boost scores for candidates whose metadata.anchor matches the query.
|
|
300
|
+
* Uses textOverlapScore (token coverage + density) — same as multi-vector blend.
|
|
301
|
+
* Short queries benefit most because anchor text is also short (≤80 chars).
|
|
302
|
+
*/
|
|
303
|
+
function applyAnchorBoost(results: RetrievalResult[], query: string): RetrievalResult[] {
|
|
304
|
+
if (results.length === 0) return results;
|
|
305
|
+
return results.map(r => {
|
|
306
|
+
const meta = parseMetadata(r.entry.metadata);
|
|
307
|
+
const anchor = meta.anchor;
|
|
308
|
+
if (typeof anchor !== "string" || !anchor) return r;
|
|
309
|
+
const overlap = textOverlapScore(query, anchor);
|
|
310
|
+
if (overlap <= 0) return r;
|
|
311
|
+
// Boost proportional to overlap quality, capped at ANCHOR_BOOST_MAX
|
|
312
|
+
const boost = 1 + overlap * ANCHOR_BOOST_MAX;
|
|
313
|
+
return { ...r, score: r.score * boost };
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Emotion scoring stage: boost memories matching query emotional tone.
|
|
319
|
+
* No-op when query is neutral or memory lacks emotion data.
|
|
320
|
+
*/
|
|
321
|
+
export function applyEmotionWeight(
|
|
322
|
+
results: Array<{ score: number; metadata: string; [k: string]: unknown }>,
|
|
323
|
+
queryEmotion: EmotionMetadata | null,
|
|
324
|
+
): typeof results {
|
|
325
|
+
if (!queryEmotion || Math.abs(queryEmotion.valence) < 0.2) {
|
|
326
|
+
return results;
|
|
327
|
+
}
|
|
328
|
+
return results.map(r => {
|
|
329
|
+
const memEmotion = parseEmotion(r.metadata);
|
|
330
|
+
if (!memEmotion) return r;
|
|
331
|
+
const alignment = 1 - Math.abs(queryEmotion.valence - memEmotion.valence) / 2;
|
|
332
|
+
const boost = 1.0 + 0.15 * alignment;
|
|
333
|
+
return { ...r, score: r.score * boost };
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// ============================================================================
|
|
338
|
+
// Source Diversity — round-robin across scopes for multi-session coverage
|
|
339
|
+
// ============================================================================
|
|
340
|
+
|
|
341
|
+
/** Minimum distinct sources to activate diversity logic. */
|
|
342
|
+
const SOURCE_DIVERSITY_MIN_SOURCES = 3;
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Ensures top-k results span multiple scopes/sessions via round-robin.
|
|
346
|
+
* Each distinct source gets its top-1 result first, remaining slots fill by score.
|
|
347
|
+
*/
|
|
348
|
+
function applySourceDiversity(results: RetrievalResult[], limit: number): RetrievalResult[] {
|
|
349
|
+
if (results.length <= limit) return results;
|
|
350
|
+
|
|
351
|
+
// Group by scope
|
|
352
|
+
const byScope = new Map<string, RetrievalResult[]>();
|
|
353
|
+
for (const r of results) {
|
|
354
|
+
const key = r.entry.scope || "__default__";
|
|
355
|
+
const bucket = byScope.get(key);
|
|
356
|
+
if (bucket) bucket.push(r);
|
|
357
|
+
else byScope.set(key, [r]);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
if (byScope.size < SOURCE_DIVERSITY_MIN_SOURCES) {
|
|
361
|
+
return results.slice(0, limit);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// Round-robin: each scope contributes its top-1
|
|
365
|
+
const selected: RetrievalResult[] = [];
|
|
366
|
+
const usedIds = new Set<string>();
|
|
367
|
+
|
|
368
|
+
// Sort scopes by their best result score (highest first)
|
|
369
|
+
const scopeEntries = [...byScope.entries()].sort(
|
|
370
|
+
(a, b) => (b[1][0]?.score ?? 0) - (a[1][0]?.score ?? 0),
|
|
371
|
+
);
|
|
372
|
+
|
|
373
|
+
for (const [, items] of scopeEntries) {
|
|
374
|
+
if (selected.length >= limit) break;
|
|
375
|
+
const top = items[0];
|
|
376
|
+
if (top && !usedIds.has(top.entry.id)) {
|
|
377
|
+
selected.push(top);
|
|
378
|
+
usedIds.add(top.entry.id);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Fill remaining slots by global score
|
|
383
|
+
for (const r of results) {
|
|
384
|
+
if (selected.length >= limit) break;
|
|
385
|
+
if (!usedIds.has(r.entry.id)) {
|
|
386
|
+
selected.push(r);
|
|
387
|
+
usedIds.add(r.entry.id);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
return selected;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// ============================================================================
|
|
395
|
+
// Adaptive Candidate Pool — detect aggregation queries
|
|
396
|
+
// ============================================================================
|
|
397
|
+
|
|
398
|
+
const AGGREGATION_EN_RE = /\b(?:how many|how much|all the|every|list all|total number|count)\b/i;
|
|
399
|
+
const AGGREGATION_CN_RE = /(?:多少[个件条只]?|一共|总共|所有的?|列举|有几[个件条只])/u;
|
|
400
|
+
|
|
401
|
+
function isAggregationQuery(query: string): boolean {
|
|
402
|
+
return AGGREGATION_EN_RE.test(query) || AGGREGATION_CN_RE.test(query);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
// ============================================================================
|
|
406
|
+
// Multi-hop Entity Extraction — extract entities from retrieval results
|
|
407
|
+
// ============================================================================
|
|
408
|
+
|
|
409
|
+
const MULTI_HOP_STOP_WORDS = new Set([
|
|
410
|
+
"the", "a", "an", "is", "are", "was", "were", "be", "been",
|
|
411
|
+
"have", "has", "had", "will", "would", "can", "could", "should",
|
|
412
|
+
"do", "does", "did", "not", "no", "but", "and", "or", "if",
|
|
413
|
+
"this", "that", "these", "those", "it", "its", "user", "assistant",
|
|
414
|
+
"my", "your", "his", "her", "our", "their", "i", "you", "he", "she",
|
|
415
|
+
"we", "they", "me", "him", "us", "them", "who", "what", "where",
|
|
416
|
+
"when", "how", "why", "which", "about", "with", "from", "for",
|
|
417
|
+
"into", "also", "just", "very", "some", "more", "most", "other",
|
|
418
|
+
]);
|
|
419
|
+
|
|
420
|
+
const COMMON_CJK_STOP = /^(用户|助手|可以|需要|已经|没有|什么|这个|那个|因为|所以|但是|如果)$/;
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Extract salient entities from retrieval result texts.
|
|
424
|
+
* Focuses on proper nouns and CJK named entities.
|
|
425
|
+
* Returns unique entities sorted by frequency (most common first).
|
|
426
|
+
*/
|
|
427
|
+
function extractEntitiesFromResults(results: RetrievalResult[]): string[] {
|
|
428
|
+
const freq = new Map<string, number>();
|
|
429
|
+
|
|
430
|
+
for (const r of results) {
|
|
431
|
+
const text = r.entry.text;
|
|
432
|
+
|
|
433
|
+
// Capitalized multi-word entities: "Adobe Premiere Pro", "Sony A7III"
|
|
434
|
+
const capPattern = /\b([A-Z][a-zA-Z0-9]*(?:[\s\-][A-Z][a-zA-Z0-9]*)*)\b/g;
|
|
435
|
+
let match: RegExpExecArray | null;
|
|
436
|
+
while ((match = capPattern.exec(text)) !== null) {
|
|
437
|
+
const entity = match[1].trim();
|
|
438
|
+
if (entity.length < 2 || MULTI_HOP_STOP_WORDS.has(entity.toLowerCase())) continue;
|
|
439
|
+
freq.set(entity, (freq.get(entity) ?? 0) + 1);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// CJK named entities: sequences of 2-8 CJK chars (simple heuristic)
|
|
443
|
+
const cjkPattern = /([\p{Script=Han}]{2,8})/gu;
|
|
444
|
+
while ((match = cjkPattern.exec(text)) !== null) {
|
|
445
|
+
const entity = match[1];
|
|
446
|
+
if (COMMON_CJK_STOP.test(entity)) continue;
|
|
447
|
+
freq.set(entity, (freq.get(entity) ?? 0) + 1);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// Sort by frequency descending, then by length descending (prefer specific entities)
|
|
452
|
+
return [...freq.entries()]
|
|
453
|
+
.sort((a, b) => b[1] - a[1] || b[0].length - a[0].length)
|
|
454
|
+
.map(([entity]) => entity);
|
|
455
|
+
}
|
|
456
|
+
|
|
183
457
|
function parseMetadata(raw?: string): Record<string, unknown> {
|
|
184
458
|
try {
|
|
185
459
|
return JSON.parse(raw || "{}") as Record<string, unknown>;
|
|
@@ -353,6 +627,9 @@ function cosineSimilarity(a: number[], b: number[]): number {
|
|
|
353
627
|
export class MemoryRetriever {
|
|
354
628
|
private accessTracker?: AccessTracker;
|
|
355
629
|
private kgStore?: KGStore;
|
|
630
|
+
private frequencyTracker?: FrequencyTracker;
|
|
631
|
+
private auditLogger?: AuditLogger;
|
|
632
|
+
private llmClient?: ReconstructionLLMClient;
|
|
356
633
|
/**
|
|
357
634
|
* Session-scoped suppression list (dopamine-inspired "do not disturb").
|
|
358
635
|
* IDs in this set get a score penalty during retrieval but are NOT deleted.
|
|
@@ -376,6 +653,21 @@ export class MemoryRetriever {
|
|
|
376
653
|
this.kgStore = kgStore;
|
|
377
654
|
}
|
|
378
655
|
|
|
656
|
+
/** P0.2: Attach a FrequencyTracker for hit-count based boosting. */
|
|
657
|
+
setFrequencyTracker(tracker: FrequencyTracker): void {
|
|
658
|
+
this.frequencyTracker = tracker;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** F-1: Attach an AuditLogger for recording retrieve operations. */
|
|
662
|
+
setAuditLogger(logger: AuditLogger): void {
|
|
663
|
+
this.auditLogger = logger;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/** Attach an LLM client for constructive retrieval reconstruction. */
|
|
667
|
+
setLLMClient(client: ReconstructionLLMClient): void {
|
|
668
|
+
this.llmClient = client;
|
|
669
|
+
}
|
|
670
|
+
|
|
379
671
|
/** Suppress a memory ID for the current session (score × 0.3). */
|
|
380
672
|
suppressForSession(id: string): void {
|
|
381
673
|
this.sessionSuppressed.add(id);
|
|
@@ -386,13 +678,15 @@ export class MemoryRetriever {
|
|
|
386
678
|
this.sessionSuppressed.clear();
|
|
387
679
|
}
|
|
388
680
|
|
|
389
|
-
/** Resolve the minimum score for a given category, falling back to hardMinScore.
|
|
390
|
-
|
|
681
|
+
/** Resolve the minimum score for a given category, falling back to hardMinScore.
|
|
682
|
+
* P0.1: When shortQuery is true, thresholds are lowered to widen the candidate pool. */
|
|
683
|
+
private minScoreFor(category: string, shortQuery = false): number {
|
|
391
684
|
const map = this.config.categoryMinScores ?? DEFAULT_CATEGORY_MIN_SCORES;
|
|
392
|
-
|
|
685
|
+
const base = map[category] ?? this.config.hardMinScore;
|
|
686
|
+
return shortQuery ? base * SHORT_QUERY_MIN_SCORE_FACTOR : base;
|
|
393
687
|
}
|
|
394
688
|
|
|
395
|
-
async retrieve(context: RetrievalContext): Promise<
|
|
689
|
+
async retrieve(context: RetrievalContext): Promise<RetrievalResultSet> {
|
|
396
690
|
const { query, limit, scopeFilter, category, includeArchived, trace, graph } = context;
|
|
397
691
|
const safeLimit = clampInt(limit, 1, 20);
|
|
398
692
|
|
|
@@ -411,18 +705,152 @@ export class MemoryRetriever {
|
|
|
411
705
|
results = await this.hybridRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace, graph);
|
|
412
706
|
}
|
|
413
707
|
|
|
708
|
+
// LME-2: Multi-hop retrieval — extract entities from first-pass results,
|
|
709
|
+
// run focused follow-up queries, merge to improve cross-session coverage.
|
|
710
|
+
const useMultiHop = context.multiHop ?? this.config.multiHop ?? false;
|
|
711
|
+
if (useMultiHop && results.length > 0) {
|
|
712
|
+
results = await this.multiHopExpand(results, context);
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
// MP-1: Topic Tag post-filter — only keep results matching the requested topicTag
|
|
716
|
+
if (context.topicTag && results.length > 0) {
|
|
717
|
+
const tag = context.topicTag.toLowerCase();
|
|
718
|
+
results = results.filter(r => {
|
|
719
|
+
const entryTag = extractTopicTag(r.entry.metadata);
|
|
720
|
+
return entryTag?.toLowerCase() === tag;
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// F3: Temporal validity filter — expired memories excluded by default
|
|
725
|
+
if (results.length > 0) {
|
|
726
|
+
results = filterByValidity(results, Date.now(), {
|
|
727
|
+
validAt: context.validAt,
|
|
728
|
+
includeExpired: context.includeExpired,
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
// P0.2: Record frequency hits for returned results (manual queries only)
|
|
733
|
+
// Moved after topicTag filter so filtered-out entries are not reinforced.
|
|
734
|
+
if (this.frequencyTracker && results.length > 0 && context.source !== "auto-recall") {
|
|
735
|
+
this.frequencyTracker.recordHits(results.map(r => r.entry.id));
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// A-3: Record evolution access counts (async, non-blocking)
|
|
739
|
+
// Moved after topicTag filter so filtered-out entries are not reinforced.
|
|
740
|
+
if (results.length > 0 && context.source !== "auto-recall" && this.store.update) {
|
|
741
|
+
Promise.resolve().then(async () => {
|
|
742
|
+
for (const r of results) {
|
|
743
|
+
try {
|
|
744
|
+
const updated = recordEvolutionAccess(r.entry.metadata);
|
|
745
|
+
await this.store.update!(r.entry.id, { metadata: updated });
|
|
746
|
+
} catch (err) { console.error("[recallnest] Evolution access tracking failed:", err instanceof Error ? err.message : String(err)); }
|
|
747
|
+
}
|
|
748
|
+
});
|
|
749
|
+
}
|
|
750
|
+
|
|
414
751
|
// Record access for returned results (async, non-blocking).
|
|
415
752
|
// Only reinforce on manual retrieval — auto-recall must not strengthen noise.
|
|
416
|
-
// Tier 3.2: pass similarity scores for novelty gating — only novel
|
|
417
|
-
// (high query-result distance) trigger access reinforcement.
|
|
753
|
+
// Tier 3.2: pass raw vector similarity scores for novelty gating — only novel
|
|
754
|
+
// retrievals (high query-result distance) trigger access reinforcement.
|
|
418
755
|
if (this.accessTracker && results.length > 0 && context.source !== "auto-recall") {
|
|
419
756
|
this.accessTracker.recordAccess(
|
|
420
757
|
results.map(r => r.entry.id),
|
|
421
|
-
results.map(r => r.score),
|
|
758
|
+
results.map(r => r.sources.vector?.score),
|
|
422
759
|
);
|
|
423
760
|
}
|
|
424
761
|
|
|
425
|
-
|
|
762
|
+
// F-1: Audit log — record retrieve operation (non-blocking, silent on failure)
|
|
763
|
+
try {
|
|
764
|
+
this.auditLogger?.log({
|
|
765
|
+
operation: "retrieve",
|
|
766
|
+
scope: context.scopeFilter?.[0],
|
|
767
|
+
actor: context.source || "manual",
|
|
768
|
+
details: `query="${context.query.slice(0, 80)}" hits=${results.length}`,
|
|
769
|
+
});
|
|
770
|
+
} catch {
|
|
771
|
+
// Audit must never block retrieval
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
// Phase 4: Constructive retrieval — first-class reconstruction (no metadata hack)
|
|
775
|
+
const resultSet = results as RetrievalResultSet;
|
|
776
|
+
const constructiveFlag = process.env.RECALLNEST_CONSTRUCTIVE_RETRIEVAL === "true";
|
|
777
|
+
if (shouldReconstruct({
|
|
778
|
+
flagEnabled: constructiveFlag,
|
|
779
|
+
callerOptIn: context.reconstruct === true,
|
|
780
|
+
resultCount: results.length,
|
|
781
|
+
llmAvailable: this.llmClient
|
|
782
|
+
? ("isAvailable" in this.llmClient ? (this.llmClient as { isAvailable(): boolean }).isAvailable() : true)
|
|
783
|
+
: false,
|
|
784
|
+
})) {
|
|
785
|
+
try {
|
|
786
|
+
const expansionDeps = this.buildExpansionDeps(context);
|
|
787
|
+
const reconstruction = await runReconstruction(
|
|
788
|
+
{ query: context.query, results, mode: "search" },
|
|
789
|
+
this.llmClient!,
|
|
790
|
+
expansionDeps,
|
|
791
|
+
);
|
|
792
|
+
resultSet.reconstruction = reconstruction;
|
|
793
|
+
} catch {
|
|
794
|
+
// Silent degradation — return raw results
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
return resultSet;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/**
|
|
802
|
+
* LME-2: Multi-hop expansion — extract entities from first-pass results,
|
|
803
|
+
* run entity-focused follow-up queries, merge with original results.
|
|
804
|
+
* Improves cross-session coverage for aggregation/counting queries.
|
|
805
|
+
*/
|
|
806
|
+
private async multiHopExpand(
|
|
807
|
+
firstPass: RetrievalResult[],
|
|
808
|
+
context: RetrievalContext,
|
|
809
|
+
): Promise<RetrievalResult[]> {
|
|
810
|
+
const maxQueries = this.config.multiHopMaxQueries ?? 3;
|
|
811
|
+
const entities = extractEntitiesFromResults(firstPass);
|
|
812
|
+
if (entities.length === 0) return firstPass;
|
|
813
|
+
|
|
814
|
+
// Pick top entities that aren't already well-covered in the query
|
|
815
|
+
const queryLower = context.query.toLowerCase();
|
|
816
|
+
const novelEntities = entities
|
|
817
|
+
.filter(e => !queryLower.includes(e.toLowerCase()))
|
|
818
|
+
.slice(0, maxQueries);
|
|
819
|
+
|
|
820
|
+
if (novelEntities.length === 0) return firstPass;
|
|
821
|
+
|
|
822
|
+
// Run follow-up queries in parallel (entity + original query context)
|
|
823
|
+
const followUpPromises = novelEntities.map(entity =>
|
|
824
|
+
this.hybridRetrieval(
|
|
825
|
+
`${entity} ${context.query}`,
|
|
826
|
+
context.limit,
|
|
827
|
+
context.scopeFilter,
|
|
828
|
+
context.category,
|
|
829
|
+
context.includeArchived,
|
|
830
|
+
undefined, // no trace for follow-up
|
|
831
|
+
context.graph,
|
|
832
|
+
).catch(() => [] as RetrievalResult[]),
|
|
833
|
+
);
|
|
834
|
+
const followUpResults = await Promise.all(followUpPromises);
|
|
835
|
+
|
|
836
|
+
// Merge: deduplicate by memory ID, keep highest score
|
|
837
|
+
const merged = new Map<string, RetrievalResult>();
|
|
838
|
+
for (const r of firstPass) {
|
|
839
|
+
merged.set(r.entry.id, r);
|
|
840
|
+
}
|
|
841
|
+
for (const batch of followUpResults) {
|
|
842
|
+
for (const r of batch) {
|
|
843
|
+
const existing = merged.get(r.entry.id);
|
|
844
|
+
if (!existing || r.score > existing.score) {
|
|
845
|
+
merged.set(r.entry.id, r);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
// Sort by score descending, limit
|
|
851
|
+
return [...merged.values()]
|
|
852
|
+
.sort((a, b) => b.score - a.score)
|
|
853
|
+
.slice(0, context.limit);
|
|
426
854
|
}
|
|
427
855
|
|
|
428
856
|
private async vectorOnlyRetrieval(
|
|
@@ -439,7 +867,12 @@ export class MemoryRetriever {
|
|
|
439
867
|
|
|
440
868
|
trace?.startStage("vector_search", 0);
|
|
441
869
|
const queryVector = await this.embedder.embedQuery(searchQuery);
|
|
442
|
-
|
|
870
|
+
// P0.1: Lower minScore for short queries
|
|
871
|
+
const shortQ = isShortQuery(searchQuery);
|
|
872
|
+
const vectorMinScore = shortQ
|
|
873
|
+
? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
|
|
874
|
+
: this.config.minScore;
|
|
875
|
+
const results = await this.store.vectorSearch(queryVector, limit, vectorMinScore, scopeFilter);
|
|
443
876
|
trace?.endStage(results.length, results.map(r => r.score));
|
|
444
877
|
|
|
445
878
|
// Filter by category if specified
|
|
@@ -464,8 +897,11 @@ export class MemoryRetriever {
|
|
|
464
897
|
},
|
|
465
898
|
} as RetrievalResult));
|
|
466
899
|
|
|
467
|
-
|
|
468
|
-
const
|
|
900
|
+
// P0.1: Anchor boost for vector-only path
|
|
901
|
+
const anchorBoosted = applyAnchorBoost(mapped, searchQuery);
|
|
902
|
+
|
|
903
|
+
trace?.startStage("recency_boost", anchorBoosted.length);
|
|
904
|
+
const boosted = this.applyRecencyBoost(anchorBoosted);
|
|
469
905
|
trace?.endStage(boosted.length, boosted.map(r => r.score));
|
|
470
906
|
|
|
471
907
|
trace?.startStage("importance_weight", boosted.length);
|
|
@@ -476,8 +912,16 @@ export class MemoryRetriever {
|
|
|
476
912
|
const confidenceWeighted = this.applyConfidence(weighted);
|
|
477
913
|
trace?.endStage(confidenceWeighted.length, confidenceWeighted.map(r => r.score));
|
|
478
914
|
|
|
479
|
-
|
|
480
|
-
|
|
915
|
+
let afterEmotionVector = confidenceWeighted;
|
|
916
|
+
if (isEmotionScoringEnabled()) {
|
|
917
|
+
const queryEmotion = detectEmotion(query);
|
|
918
|
+
const adapted = confidenceWeighted.map(r => ({ ...r, metadata: r.entry.metadata ?? "{}" }));
|
|
919
|
+
const emotioned = applyEmotionWeight(adapted, queryEmotion);
|
|
920
|
+
afterEmotionVector = confidenceWeighted.map((r, i) => ({ ...r, score: emotioned[i]?.score ?? r.score }));
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
trace?.startStage("boundary_weight", afterEmotionVector.length);
|
|
924
|
+
const boundaryWeighted = this.applyBoundaryWeight(afterEmotionVector);
|
|
481
925
|
trace?.endStage(boundaryWeighted.length, boundaryWeighted.map(r => r.score));
|
|
482
926
|
|
|
483
927
|
trace?.startStage("asset_type_weight", boundaryWeighted.length);
|
|
@@ -492,22 +936,39 @@ export class MemoryRetriever {
|
|
|
492
936
|
const timeDecayed = this.applyTimeDecay(lengthNormalized);
|
|
493
937
|
trace?.endStage(timeDecayed.length, timeDecayed.map(r => r.score));
|
|
494
938
|
|
|
939
|
+
// B-1/E-1: Evolution decay blend — boost memories with high composite decay score
|
|
940
|
+
const evolutionBlended = this.applyEvolutionDecayBlend(timeDecayed);
|
|
941
|
+
|
|
942
|
+
// E-1: Access count boost — memories retrieved more often get a score nudge
|
|
943
|
+
const accessBoosted = this.applyAccessCountBoost(evolutionBlended);
|
|
944
|
+
|
|
495
945
|
// Hotness blend: boost frequently + recently accessed memories
|
|
496
|
-
trace?.startStage("hotness_blend",
|
|
497
|
-
const hotnessBlended = this.applyHotnessBlend(
|
|
946
|
+
trace?.startStage("hotness_blend", accessBoosted.length);
|
|
947
|
+
const hotnessBlended = this.applyHotnessBlend(accessBoosted);
|
|
498
948
|
trace?.endStage(hotnessBlended.length, hotnessBlended.map(r => r.score));
|
|
499
949
|
|
|
950
|
+
// P0.2: Frequency boost — repeatedly retrieved memories score higher
|
|
951
|
+
trace?.startStage("frequency_boost", hotnessBlended.length);
|
|
952
|
+
const frequencyBoosted = this.applyFrequencyBoost(hotnessBlended);
|
|
953
|
+
trace?.endStage(frequencyBoosted.length, frequencyBoosted.map(r => r.score));
|
|
954
|
+
|
|
500
955
|
// Session suppression: temporarily penalize "do not disturb" memories
|
|
501
|
-
const afterSuppression = this.applySessionSuppression(
|
|
956
|
+
const afterSuppression = this.applySessionSuppression(frequencyBoosted);
|
|
957
|
+
const rescored = [...afterSuppression].sort((a, b) => b.score - a.score);
|
|
502
958
|
|
|
503
|
-
trace?.startStage("hard_min_score",
|
|
504
|
-
const hardFiltered =
|
|
959
|
+
trace?.startStage("hard_min_score", rescored.length);
|
|
960
|
+
const hardFiltered = rescored.filter(r => r.score >= this.minScoreFor(r.entry.category, shortQ));
|
|
505
961
|
trace?.endStage(hardFiltered.length, hardFiltered.map(r => r.score));
|
|
506
962
|
|
|
507
|
-
|
|
963
|
+
// Evolution status filter: exclude superseded/archived/consolidated memories
|
|
964
|
+
const evolutionFiltered = includeArchived
|
|
965
|
+
? hardFiltered
|
|
966
|
+
: hardFiltered.filter(r => isActiveMemory(r.entry.metadata));
|
|
967
|
+
|
|
968
|
+
trace?.startStage("noise_filter", evolutionFiltered.length);
|
|
508
969
|
const denoised = this.config.filterNoise
|
|
509
|
-
? filterNoise(
|
|
510
|
-
:
|
|
970
|
+
? filterNoise(evolutionFiltered, r => r.entry.text)
|
|
971
|
+
: evolutionFiltered;
|
|
511
972
|
trace?.endStage(denoised.length, denoised.map(r => r.score));
|
|
512
973
|
|
|
513
974
|
// RIF: demote near-duplicate weak results to improve diversity
|
|
@@ -523,8 +984,16 @@ export class MemoryRetriever {
|
|
|
523
984
|
// Tier 3.3: Version group dedup — keep only the top-ranked version per group
|
|
524
985
|
const versionDeduped = deduplicateByVersionGroup(deduplicated);
|
|
525
986
|
|
|
526
|
-
|
|
527
|
-
const
|
|
987
|
+
// LC-P2: Cluster insight dedup — prefer cluster summary over individual source memories
|
|
988
|
+
const clusterDeduped = deduplicateByClusterInsight(versionDeduped);
|
|
989
|
+
|
|
990
|
+
// Source diversity: round-robin across scopes for multi-session coverage
|
|
991
|
+
const diversified = (this.config.sourceDiversity ?? 0) > 0
|
|
992
|
+
? applySourceDiversity(clusterDeduped, limit)
|
|
993
|
+
: clusterDeduped;
|
|
994
|
+
|
|
995
|
+
trace?.startStage("final_limit", diversified.length);
|
|
996
|
+
const final = diversified.slice(0, limit);
|
|
528
997
|
trace?.endStage(final.length, final.map(r => r.score));
|
|
529
998
|
|
|
530
999
|
return final;
|
|
@@ -539,7 +1008,14 @@ export class MemoryRetriever {
|
|
|
539
1008
|
trace?: TraceCollector,
|
|
540
1009
|
graph?: boolean,
|
|
541
1010
|
): Promise<RetrievalResult[]> {
|
|
542
|
-
|
|
1011
|
+
// Adaptive pool: widen candidate pool for aggregation queries ("how many", "all the")
|
|
1012
|
+
const multiplier = (this.config.adaptivePoolMultiplier ?? 1) > 1 && isAggregationQuery(query)
|
|
1013
|
+
? (this.config.adaptivePoolMultiplier ?? 1)
|
|
1014
|
+
: 1;
|
|
1015
|
+
const candidatePoolSize = Math.max(
|
|
1016
|
+
Math.ceil(this.config.candidatePoolSize * multiplier),
|
|
1017
|
+
limit * 2,
|
|
1018
|
+
);
|
|
543
1019
|
|
|
544
1020
|
// Temporal reasoning: extract time constraint and clean query for semantic search
|
|
545
1021
|
const temporal = parseTemporalQuery(query);
|
|
@@ -554,10 +1030,14 @@ export class MemoryRetriever {
|
|
|
554
1030
|
? this.runPPRSearch(query, scopeFilter?.[0], trace)
|
|
555
1031
|
: Promise.resolve([] as RetrievalResult[]);
|
|
556
1032
|
|
|
1033
|
+
// Pre-tokenize expanded query for BM25 so CJK text produces meaningful FTS matches
|
|
1034
|
+
const expandedQuery = expandQuery(searchQuery);
|
|
1035
|
+
const ftsQuery = tokenizeFts(expandedQuery, detectLang(expandedQuery));
|
|
1036
|
+
|
|
557
1037
|
trace?.startStage("vector_search", 0);
|
|
558
1038
|
const [vectorResults, bm25Results, pprResults] = await Promise.all([
|
|
559
1039
|
this.runVectorSearch(queryVector, candidatePoolSize, scopeFilter, category, includeArchived),
|
|
560
|
-
this.runBM25Search(
|
|
1040
|
+
this.runBM25Search(ftsQuery, candidatePoolSize, scopeFilter, category, includeArchived),
|
|
561
1041
|
pprPromise,
|
|
562
1042
|
]);
|
|
563
1043
|
trace?.endStage(vectorResults.length, vectorResults.map(r => r.score));
|
|
@@ -565,9 +1045,10 @@ export class MemoryRetriever {
|
|
|
565
1045
|
trace?.startStage("bm25_search", 0);
|
|
566
1046
|
trace?.endStage(bm25Results.length, bm25Results.map(r => r.score));
|
|
567
1047
|
|
|
568
|
-
// Fuse results using
|
|
1048
|
+
// Fuse results using weighted score fusion (async: validates BM25-only entries exist in store)
|
|
1049
|
+
const shortQ = isShortQuery(searchQuery);
|
|
569
1050
|
trace?.startStage("rrf_fusion", vectorResults.length + bm25Results.length + pprResults.length);
|
|
570
|
-
const fusedResults = await this.fuseResults(vectorResults, bm25Results, pprResults);
|
|
1051
|
+
const fusedResults = await this.fuseResults(vectorResults, bm25Results, pprResults, shortQ);
|
|
571
1052
|
trace?.endStage(fusedResults.length, fusedResults.map(r => r.score));
|
|
572
1053
|
|
|
573
1054
|
// Temporal filter: when query has a time constraint, remove non-matching candidates
|
|
@@ -582,9 +1063,18 @@ export class MemoryRetriever {
|
|
|
582
1063
|
const multiVecBlended = this.applyMultiVectorBlend(temporalFiltered, searchQuery);
|
|
583
1064
|
trace?.endStage(multiVecBlended.length, multiVecBlended.map(r => r.score));
|
|
584
1065
|
|
|
1066
|
+
// P0.1: Anchor boost — short query terms matching short anchor text
|
|
1067
|
+
trace?.startStage("anchor_boost", multiVecBlended.length);
|
|
1068
|
+
const anchorBoosted = applyAnchorBoost(multiVecBlended, searchQuery);
|
|
1069
|
+
trace?.endStage(anchorBoosted.length, anchorBoosted.map(r => r.score));
|
|
1070
|
+
|
|
585
1071
|
// Apply minimum score threshold
|
|
586
|
-
trace?.startStage("min_score_filter",
|
|
587
|
-
|
|
1072
|
+
trace?.startStage("min_score_filter", anchorBoosted.length);
|
|
1073
|
+
// P0.1: Lower minScore for short queries to widen the candidate pool
|
|
1074
|
+
const effectiveMinScore = isShortQuery(searchQuery)
|
|
1075
|
+
? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
|
|
1076
|
+
: this.config.minScore;
|
|
1077
|
+
const filtered = anchorBoosted.filter(r => r.score >= effectiveMinScore);
|
|
588
1078
|
trace?.endStage(filtered.length, filtered.map(r => r.score));
|
|
589
1079
|
|
|
590
1080
|
// Rerank if enabled
|
|
@@ -609,9 +1099,18 @@ export class MemoryRetriever {
|
|
|
609
1099
|
const confidenceWeighted = this.applyConfidence(importanceWeighted);
|
|
610
1100
|
trace?.endStage(confidenceWeighted.length, confidenceWeighted.map(r => r.score));
|
|
611
1101
|
|
|
1102
|
+
// Emotion scoring: boost memories matching query emotional tone
|
|
1103
|
+
let afterEmotion = confidenceWeighted;
|
|
1104
|
+
if (isEmotionScoringEnabled()) {
|
|
1105
|
+
const queryEmotion = detectEmotion(query);
|
|
1106
|
+
const adapted = confidenceWeighted.map(r => ({ ...r, metadata: r.entry.metadata ?? "{}" }));
|
|
1107
|
+
const emotioned = applyEmotionWeight(adapted, queryEmotion);
|
|
1108
|
+
afterEmotion = confidenceWeighted.map((r, i) => ({ ...r, score: emotioned[i]?.score ?? r.score }));
|
|
1109
|
+
}
|
|
1110
|
+
|
|
612
1111
|
// Prefer higher-authority durable memories over raw evidence when scores are close.
|
|
613
|
-
trace?.startStage("boundary_weight",
|
|
614
|
-
const boundaryWeighted = this.applyBoundaryWeight(
|
|
1112
|
+
trace?.startStage("boundary_weight", afterEmotion.length);
|
|
1113
|
+
const boundaryWeighted = this.applyBoundaryWeight(afterEmotion);
|
|
615
1114
|
trace?.endStage(boundaryWeighted.length, boundaryWeighted.map(r => r.score));
|
|
616
1115
|
|
|
617
1116
|
// Separate pinned assets from synthesized briefs.
|
|
@@ -629,21 +1128,40 @@ export class MemoryRetriever {
|
|
|
629
1128
|
const timeDecayed = this.applyTimeDecay(lengthNormalized);
|
|
630
1129
|
trace?.endStage(timeDecayed.length, timeDecayed.map(r => r.score));
|
|
631
1130
|
|
|
1131
|
+
// B-1/E-1: Evolution decay blend — boost memories with high composite decay score
|
|
1132
|
+
const evolutionBlended = this.applyEvolutionDecayBlend(timeDecayed);
|
|
1133
|
+
|
|
1134
|
+
// E-1: Access count boost — memories retrieved more often get a score nudge
|
|
1135
|
+
const accessBoosted = this.applyAccessCountBoost(evolutionBlended);
|
|
1136
|
+
|
|
632
1137
|
// Hotness blend: boost frequently + recently accessed memories
|
|
633
|
-
trace?.startStage("hotness_blend",
|
|
634
|
-
const hotnessBlended = this.applyHotnessBlend(
|
|
1138
|
+
trace?.startStage("hotness_blend", accessBoosted.length);
|
|
1139
|
+
const hotnessBlended = this.applyHotnessBlend(accessBoosted);
|
|
635
1140
|
trace?.endStage(hotnessBlended.length, hotnessBlended.map(r => r.score));
|
|
636
1141
|
|
|
1142
|
+
// P0.2: Frequency boost — repeatedly retrieved memories score higher
|
|
1143
|
+
trace?.startStage("frequency_boost", hotnessBlended.length);
|
|
1144
|
+
const frequencyBoosted = this.applyFrequencyBoost(hotnessBlended);
|
|
1145
|
+
trace?.endStage(frequencyBoosted.length, frequencyBoosted.map(r => r.score));
|
|
1146
|
+
|
|
637
1147
|
// Hard minimum score cutoff (post all scoring stages)
|
|
638
|
-
|
|
639
|
-
const
|
|
1148
|
+
// P0.1: lower thresholds for short queries (shortQ declared earlier for fuseResults)
|
|
1149
|
+
const rescored = [...frequencyBoosted].sort((a, b) => b.score - a.score);
|
|
1150
|
+
|
|
1151
|
+
trace?.startStage("hard_min_score", rescored.length);
|
|
1152
|
+
const hardFiltered = rescored.filter(r => r.score >= this.minScoreFor(r.entry.category, shortQ));
|
|
640
1153
|
trace?.endStage(hardFiltered.length, hardFiltered.map(r => r.score));
|
|
641
1154
|
|
|
1155
|
+
// Evolution status filter: exclude superseded/archived/consolidated memories
|
|
1156
|
+
const evolutionFiltered = includeArchived
|
|
1157
|
+
? hardFiltered
|
|
1158
|
+
: hardFiltered.filter(r => isActiveMemory(r.entry.metadata));
|
|
1159
|
+
|
|
642
1160
|
// Filter noise
|
|
643
|
-
trace?.startStage("noise_filter",
|
|
1161
|
+
trace?.startStage("noise_filter", evolutionFiltered.length);
|
|
644
1162
|
const denoised = this.config.filterNoise
|
|
645
|
-
? filterNoise(
|
|
646
|
-
:
|
|
1163
|
+
? filterNoise(evolutionFiltered, r => r.entry.text)
|
|
1164
|
+
: evolutionFiltered;
|
|
647
1165
|
trace?.endStage(denoised.length, denoised.map(r => r.score));
|
|
648
1166
|
|
|
649
1167
|
// RIF: demote near-duplicate weak results to improve diversity
|
|
@@ -659,8 +1177,16 @@ export class MemoryRetriever {
|
|
|
659
1177
|
// Tier 3.3: Version group dedup — keep only the top-ranked version per group
|
|
660
1178
|
const versionDeduped = deduplicateByVersionGroup(deduplicated);
|
|
661
1179
|
|
|
662
|
-
|
|
663
|
-
const
|
|
1180
|
+
// LC-P2: Cluster insight dedup — prefer cluster summary over individual source memories
|
|
1181
|
+
const clusterDeduped = deduplicateByClusterInsight(versionDeduped);
|
|
1182
|
+
|
|
1183
|
+
// Source diversity: round-robin across scopes for multi-session coverage
|
|
1184
|
+
const diversified = (this.config.sourceDiversity ?? 0) > 0
|
|
1185
|
+
? applySourceDiversity(clusterDeduped, limit)
|
|
1186
|
+
: clusterDeduped;
|
|
1187
|
+
|
|
1188
|
+
trace?.startStage("final_limit", diversified.length);
|
|
1189
|
+
const final = diversified.slice(0, limit);
|
|
664
1190
|
trace?.endStage(final.length, final.map(r => r.score));
|
|
665
1191
|
|
|
666
1192
|
return final;
|
|
@@ -806,7 +1332,18 @@ export class MemoryRetriever {
|
|
|
806
1332
|
vectorResults: Array<MemorySearchResult & { rank: number }>,
|
|
807
1333
|
bm25Results: Array<MemorySearchResult & { rank: number }>,
|
|
808
1334
|
graphResults: RetrievalResult[] = [],
|
|
1335
|
+
shortQuery: boolean = false,
|
|
809
1336
|
): Promise<RetrievalResult[]> {
|
|
1337
|
+
// Adaptive weights: short queries (≤4 tokens) favor BM25 for exact keyword matching.
|
|
1338
|
+
// For normal queries, use config weights directly.
|
|
1339
|
+
const vW = shortQuery
|
|
1340
|
+
? this.config.vectorWeight * 0.7
|
|
1341
|
+
: this.config.vectorWeight;
|
|
1342
|
+
const bW = shortQuery
|
|
1343
|
+
? Math.min(this.config.bm25Weight * 1.5, 0.6)
|
|
1344
|
+
: this.config.bm25Weight;
|
|
1345
|
+
const totalW = vW + bW;
|
|
1346
|
+
|
|
810
1347
|
// Create maps for quick lookup
|
|
811
1348
|
const vectorMap = new Map<string, MemorySearchResult & { rank: number }>();
|
|
812
1349
|
const bm25Map = new Map<string, MemorySearchResult & { rank: number }>();
|
|
@@ -827,7 +1364,7 @@ export class MemoryRetriever {
|
|
|
827
1364
|
// Get all unique document IDs
|
|
828
1365
|
const allIds = new Set([...vectorMap.keys(), ...bm25Map.keys(), ...graphMap.keys()]);
|
|
829
1366
|
|
|
830
|
-
// Calculate
|
|
1367
|
+
// Calculate fused scores using configurable weights
|
|
831
1368
|
const fusedResults: RetrievalResult[] = [];
|
|
832
1369
|
|
|
833
1370
|
for (const id of allIds) {
|
|
@@ -850,26 +1387,29 @@ export class MemoryRetriever {
|
|
|
850
1387
|
// Use the result with more complete data (prefer vector > graph > BM25)
|
|
851
1388
|
const baseResult = vectorResult || graphResult || bm25Result!;
|
|
852
1389
|
|
|
853
|
-
// Use vector similarity as the base score.
|
|
854
|
-
// BM25 hit acts as a bonus (keyword match confirms relevance).
|
|
855
|
-
// Graph hit acts as a bonus (entity relationship confirms relevance).
|
|
856
1390
|
const vectorScore = vectorResult ? vectorResult.score : 0;
|
|
857
|
-
const
|
|
1391
|
+
const bm25Score = bm25Result ? bm25Result.score : 0;
|
|
858
1392
|
const graphScore = graphResult?.sources.graph?.score ?? 0;
|
|
859
1393
|
|
|
860
|
-
// Base = vector score; BM25 hit boosts by up to 15%; graph boosts by up to 20%
|
|
861
|
-
// Graph-only results use their PPR score so entity-connected memories surface.
|
|
862
1394
|
let fusedScore: number;
|
|
863
|
-
if (vectorResult) {
|
|
1395
|
+
if (vectorResult && bm25Result) {
|
|
1396
|
+
// Both signals agree: weighted average + 5% dual-match confirmation bonus.
|
|
1397
|
+
// Graph hit adds up to 15% on top.
|
|
864
1398
|
fusedScore = clamp01(
|
|
865
|
-
vectorScore +
|
|
1399
|
+
((vW * vectorScore + bW * bm25Score) / totalW) * 1.05 + (graphScore * 0.15),
|
|
866
1400
|
0.1,
|
|
867
1401
|
);
|
|
1402
|
+
} else if (vectorResult) {
|
|
1403
|
+
// Vector only: semantic match, graph as bonus
|
|
1404
|
+
fusedScore = clamp01(vectorScore + (graphScore * 0.15), 0.1);
|
|
868
1405
|
} else if (graphResult) {
|
|
869
|
-
// Graph
|
|
870
|
-
fusedScore = clamp01(graphScore + (
|
|
1406
|
+
// Graph only: entity-relationship match, BM25 as bonus
|
|
1407
|
+
fusedScore = clamp01(graphScore + (bm25Score > 0 ? 0.10 : 0), 0.1);
|
|
871
1408
|
} else {
|
|
872
|
-
|
|
1409
|
+
// BM25 only: exact keyword match without semantic confirmation.
|
|
1410
|
+
// Short queries get a small boost since BM25 is more reliable for exact terms.
|
|
1411
|
+
const bm25OnlyBoost = shortQuery ? 1.15 : 1.0;
|
|
1412
|
+
fusedScore = clamp01(bm25Score * bm25OnlyBoost, 0.1);
|
|
873
1413
|
}
|
|
874
1414
|
|
|
875
1415
|
fusedResults.push({
|
|
@@ -1178,22 +1718,33 @@ export class MemoryRetriever {
|
|
|
1178
1718
|
|
|
1179
1719
|
const now = Date.now();
|
|
1180
1720
|
const decayed = results.map(r => {
|
|
1721
|
+
// HP-7: Exempt core/pinned/recently-accessed from decay
|
|
1722
|
+
if (isDecayExempt(r.entry.metadata, r.entry.importance)) return r;
|
|
1723
|
+
|
|
1181
1724
|
const ts = (r.entry.timestamp && r.entry.timestamp > 0) ? r.entry.timestamp : now;
|
|
1182
1725
|
const ageDays = (now - ts) / 86_400_000;
|
|
1183
1726
|
|
|
1184
1727
|
// Use access-reinforced half-life if tracker is attached
|
|
1185
|
-
|
|
1728
|
+
let halfLife = this.accessTracker
|
|
1186
1729
|
? this.accessTracker.computeEffectiveHalfLife(baseHalfLife, r.entry.metadata)
|
|
1187
1730
|
: baseHalfLife;
|
|
1188
1731
|
|
|
1732
|
+
// HP-emo: Emotion-adjusted half-life — strong emotion slows forgetting
|
|
1733
|
+
const emotion = isEmotionScoringEnabled() ? parseEmotion(r.entry.metadata) : null;
|
|
1734
|
+
if (emotion) {
|
|
1735
|
+
halfLife = adjustHalfLifeForEmotion(halfLife, emotion);
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1189
1738
|
// Resolve tier from metadata (defaults to peripheral)
|
|
1190
1739
|
const tier = resolveTier(r.entry.metadata);
|
|
1191
1740
|
|
|
1192
1741
|
// Weibull decay with tier-specific shape and floor
|
|
1193
1742
|
const factor = weibullDecay(ageDays, halfLife, tier);
|
|
1743
|
+
// HP-emo: Arousal boost — flashbulb memory effect (1.0–1.1x)
|
|
1744
|
+
const arousalFactor = emotion ? computeArousalBoost(emotion) : 1.0;
|
|
1194
1745
|
return {
|
|
1195
1746
|
...r,
|
|
1196
|
-
score: clamp01(r.score * factor, r.score * 0.3),
|
|
1747
|
+
score: clamp01(r.score * arousalFactor * factor, r.score * 0.3),
|
|
1197
1748
|
};
|
|
1198
1749
|
});
|
|
1199
1750
|
|
|
@@ -1208,6 +1759,214 @@ export class MemoryRetriever {
|
|
|
1208
1759
|
*
|
|
1209
1760
|
* No-op when hotnessWeight is 0 or AccessTracker is absent.
|
|
1210
1761
|
*/
|
|
1762
|
+
/**
|
|
1763
|
+
* B-1/E-1: Evolution decay blend — factor in composite decay score
|
|
1764
|
+
* (time × frequency × importance) as a scoring signal.
|
|
1765
|
+
* Weight increased from 0.05 to 0.10 with B-3 LLM importance assessment.
|
|
1766
|
+
*/
|
|
1767
|
+
private applyEvolutionDecayBlend(results: RetrievalResult[]): RetrievalResult[] {
|
|
1768
|
+
const EVOLUTION_BLEND_WEIGHT = 0.10;
|
|
1769
|
+
return results.map(r => {
|
|
1770
|
+
// HP-7: Exempt entries get max decay score (1.0) → no penalty
|
|
1771
|
+
if (isDecayExempt(r.entry.metadata, r.entry.importance)) {
|
|
1772
|
+
return {
|
|
1773
|
+
...r,
|
|
1774
|
+
score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + 1.0 * EVOLUTION_BLEND_WEIGHT, 0),
|
|
1775
|
+
};
|
|
1776
|
+
}
|
|
1777
|
+
const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
|
|
1778
|
+
const decay = computeDecayScore(evo, r.entry.importance, undefined, r.entry.metadata);
|
|
1779
|
+
return {
|
|
1780
|
+
...r,
|
|
1781
|
+
score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + decay * EVOLUTION_BLEND_WEIGHT, 0),
|
|
1782
|
+
};
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
/**
|
|
1787
|
+
* E-1: Access count boost — memories with higher access_count get a score nudge.
|
|
1788
|
+
* Complements frequency-tracker (query→memory mapping) by operating at
|
|
1789
|
+
* the individual memory level (total retrieval hits across all queries).
|
|
1790
|
+
* Formula: score *= 1 + log2(1 + accessCount) × 0.03, capped at ×1.15.
|
|
1791
|
+
*/
|
|
1792
|
+
private applyAccessCountBoost(results: RetrievalResult[]): RetrievalResult[] {
|
|
1793
|
+
return results.map(r => {
|
|
1794
|
+
const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
|
|
1795
|
+
if (evo.accessCount <= 0) return r;
|
|
1796
|
+
const boost = Math.min(0.15, Math.log2(1 + evo.accessCount) * 0.03);
|
|
1797
|
+
return {
|
|
1798
|
+
...r,
|
|
1799
|
+
score: clamp01(r.score * (1 + boost), 0),
|
|
1800
|
+
};
|
|
1801
|
+
});
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
/**
|
|
1805
|
+
* Phase 4: Build candidate expansion deps from internal stores.
|
|
1806
|
+
* Each dep is only provided if the corresponding store/feature is available.
|
|
1807
|
+
*/
|
|
1808
|
+
private buildExpansionDeps(context: RetrievalContext): CandidateExpansionDeps {
|
|
1809
|
+
const deps: CandidateExpansionDeps = {};
|
|
1810
|
+
const scope = context.scopeFilter?.[0];
|
|
1811
|
+
|
|
1812
|
+
// KG neighbor expansion — requires KG mode + attached KG store
|
|
1813
|
+
if (isKGModeEnabled() && this.kgStore) {
|
|
1814
|
+
const kgStore = this.kgStore;
|
|
1815
|
+
const store = this.store;
|
|
1816
|
+
deps.expandViaKG = async (results) => {
|
|
1817
|
+
// Extract unique source_memory_ids from KG triples linked to result entities
|
|
1818
|
+
const entityNames: string[] = [];
|
|
1819
|
+
for (const r of results.slice(0, 5)) {
|
|
1820
|
+
try {
|
|
1821
|
+
const meta = JSON.parse(r.entry.metadata || "{}");
|
|
1822
|
+
if (meta.kg_entities && Array.isArray(meta.kg_entities)) {
|
|
1823
|
+
entityNames.push(...meta.kg_entities);
|
|
1824
|
+
}
|
|
1825
|
+
} catch (err) { console.error("[recallnest] KG entity extraction failed:", err instanceof Error ? err.message : String(err)); }
|
|
1826
|
+
}
|
|
1827
|
+
if (entityNames.length === 0) return [];
|
|
1828
|
+
const unique = [...new Set(entityNames)].slice(0, 5);
|
|
1829
|
+
const neighborhood = await kgStore.getNeighborhood(unique, 1, scope);
|
|
1830
|
+
const sourceIds = new Set<string>();
|
|
1831
|
+
const existingIds = new Set(results.map(r => r.entry.id));
|
|
1832
|
+
for (const n of neighborhood) {
|
|
1833
|
+
for (const t of n.triples) {
|
|
1834
|
+
if (t.source_memory_id && !existingIds.has(t.source_memory_id)) {
|
|
1835
|
+
sourceIds.add(t.source_memory_id);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
// Fetch entries by ID
|
|
1840
|
+
const expanded: RetrievalResult[] = [];
|
|
1841
|
+
for (const id of [...sourceIds].slice(0, 5)) {
|
|
1842
|
+
const entry = await store.getById(id);
|
|
1843
|
+
if (entry) {
|
|
1844
|
+
expanded.push({ entry, score: 0.5, sources: {} });
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
return expanded;
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
// Evolution chain expansion — walk supersedes/supersededBy links
|
|
1852
|
+
if (this.store.getById) {
|
|
1853
|
+
const store = this.store;
|
|
1854
|
+
deps.expandViaEvolution = async (results) => {
|
|
1855
|
+
const expanded: RetrievalResult[] = [];
|
|
1856
|
+
const existingIds = new Set(results.map(r => r.entry.id));
|
|
1857
|
+
for (const r of results.slice(0, 5)) {
|
|
1858
|
+
const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
|
|
1859
|
+
for (const linkedId of [evo.supersedes, evo.supersededBy].filter(Boolean) as string[]) {
|
|
1860
|
+
if (existingIds.has(linkedId)) continue;
|
|
1861
|
+
const entry = await store.getById(linkedId);
|
|
1862
|
+
if (entry) {
|
|
1863
|
+
existingIds.add(linkedId);
|
|
1864
|
+
expanded.push({ entry, score: r.score * 0.6, sources: {} });
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
return expanded;
|
|
1869
|
+
};
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
// Cluster member expansion — get sourceMemories from cluster insights
|
|
1873
|
+
if (this.store.getById) {
|
|
1874
|
+
const store = this.store;
|
|
1875
|
+
deps.expandViaClusters = async (results) => {
|
|
1876
|
+
const expanded: RetrievalResult[] = [];
|
|
1877
|
+
const existingIds = new Set(results.map(r => r.entry.id));
|
|
1878
|
+
for (const r of results.slice(0, 5)) {
|
|
1879
|
+
try {
|
|
1880
|
+
const meta = JSON.parse(r.entry.metadata || "{}");
|
|
1881
|
+
if ((meta.cluster_insight || meta.cross_memory_pattern) && meta.evolution?.sourceMemories) {
|
|
1882
|
+
for (const memberId of (meta.evolution.sourceMemories as string[]).slice(0, 3)) {
|
|
1883
|
+
if (existingIds.has(memberId)) continue;
|
|
1884
|
+
const entry = await store.getById(memberId);
|
|
1885
|
+
if (entry) {
|
|
1886
|
+
existingIds.add(memberId);
|
|
1887
|
+
expanded.push({ entry, score: r.score * 0.5, sources: {} });
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
} catch (err) { console.error("[recallnest] Cluster member expansion failed:", err instanceof Error ? err.message : String(err)); }
|
|
1892
|
+
}
|
|
1893
|
+
return expanded;
|
|
1894
|
+
};
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
// Narrative sibling expansion — reuse existing expandNarrativeSiblings logic
|
|
1898
|
+
if (isNarrativeModeEnabled()) {
|
|
1899
|
+
deps.expandViaNarrative = async (results) => {
|
|
1900
|
+
const expanded = await this.expandNarrativeSiblings(results);
|
|
1901
|
+
// Return only the new siblings (not in original results)
|
|
1902
|
+
const existingIds = new Set(results.map(r => r.entry.id));
|
|
1903
|
+
return expanded.filter(r => !existingIds.has(r.entry.id));
|
|
1904
|
+
};
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
return deps;
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
/**
|
|
1911
|
+
* HP-narrative: Expand retrieval results with narrative siblings.
|
|
1912
|
+
* When a retrieved memory has narrative metadata, pull other memories
|
|
1913
|
+
* sharing the same generalEventId to provide temporal context.
|
|
1914
|
+
*
|
|
1915
|
+
* Siblings are added with a dampened score (70% of the triggering result)
|
|
1916
|
+
* and capped at 3 siblings per general event to avoid flooding.
|
|
1917
|
+
*/
|
|
1918
|
+
async expandNarrativeSiblings(results: RetrievalResult[]): Promise<RetrievalResult[]> {
|
|
1919
|
+
if (!isNarrativeModeEnabled() || results.length === 0) return results;
|
|
1920
|
+
|
|
1921
|
+
// Collect unique generalEventIds from results
|
|
1922
|
+
const eventIdToScore = new Map<string, number>();
|
|
1923
|
+
const existingIds = new Set(results.map(r => r.entry.id));
|
|
1924
|
+
|
|
1925
|
+
for (const r of results) {
|
|
1926
|
+
const narrative = parseNarrative(r.entry.metadata);
|
|
1927
|
+
if (!narrative) continue;
|
|
1928
|
+
const existing = eventIdToScore.get(narrative.generalEventId);
|
|
1929
|
+
if (!existing || r.score > existing) {
|
|
1930
|
+
eventIdToScore.set(narrative.generalEventId, r.score);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
if (eventIdToScore.size === 0) return results;
|
|
1935
|
+
|
|
1936
|
+
// Search store for siblings — use scope-based listing and filter by generalEventId
|
|
1937
|
+
const siblings: RetrievalResult[] = [];
|
|
1938
|
+
const MAX_SIBLINGS_PER_EVENT = 3;
|
|
1939
|
+
|
|
1940
|
+
try {
|
|
1941
|
+
const allEntries = await this.store.list(undefined, undefined, 500, 0);
|
|
1942
|
+
for (const entry of allEntries) {
|
|
1943
|
+
if (existingIds.has(entry.id)) continue;
|
|
1944
|
+
const narrative = parseNarrative(entry.metadata);
|
|
1945
|
+
if (!narrative) continue;
|
|
1946
|
+
const triggerScore = eventIdToScore.get(narrative.generalEventId);
|
|
1947
|
+
if (triggerScore === undefined) continue;
|
|
1948
|
+
|
|
1949
|
+
// Count how many siblings we've already added for this event
|
|
1950
|
+
const eventSiblingCount = siblings.filter(s => {
|
|
1951
|
+
const sn = parseNarrative(s.entry.metadata);
|
|
1952
|
+
return sn?.generalEventId === narrative.generalEventId;
|
|
1953
|
+
}).length;
|
|
1954
|
+
if (eventSiblingCount >= MAX_SIBLINGS_PER_EVENT) continue;
|
|
1955
|
+
|
|
1956
|
+
siblings.push({
|
|
1957
|
+
entry,
|
|
1958
|
+
score: triggerScore * 0.7,
|
|
1959
|
+
sources: { narrativeSibling: true },
|
|
1960
|
+
});
|
|
1961
|
+
existingIds.add(entry.id);
|
|
1962
|
+
}
|
|
1963
|
+
} catch {
|
|
1964
|
+
// Non-blocking — if store listing fails, return original results
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
return [...results, ...siblings].sort((a, b) => b.score - a.score);
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1211
1970
|
private applyHotnessBlend(results: RetrievalResult[]): RetrievalResult[] {
|
|
1212
1971
|
const raw = this.config.hotnessWeight;
|
|
1213
1972
|
const alpha = Math.min(1, Math.max(0, Number.isFinite(raw) ? raw : 0));
|
|
@@ -1228,6 +1987,20 @@ export class MemoryRetriever {
|
|
|
1228
1987
|
return blended.sort((a, b) => b.score - a.score);
|
|
1229
1988
|
}
|
|
1230
1989
|
|
|
1990
|
+
/**
|
|
1991
|
+
* P0.2: Frequency boost — memories that get repeatedly retrieved score higher.
|
|
1992
|
+
* Unlike hotness (access-tracker based, decay-oriented), this is pure hit-count
|
|
1993
|
+
* boost designed to surface "always relevant" memories like daily patrol rules.
|
|
1994
|
+
*/
|
|
1995
|
+
private applyFrequencyBoost(results: RetrievalResult[]): RetrievalResult[] {
|
|
1996
|
+
if (!this.frequencyTracker || results.length === 0) return results;
|
|
1997
|
+
return results.map(r => {
|
|
1998
|
+
const multiplier = this.frequencyTracker!.getBoostMultiplier(r.entry.id);
|
|
1999
|
+
if (multiplier <= 1.0) return r;
|
|
2000
|
+
return { ...r, score: Math.min(1, r.score * multiplier) };
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
|
|
1231
2004
|
/**
|
|
1232
2005
|
* MMR-inspired diversity filter: greedily select results that are both
|
|
1233
2006
|
* relevant (high score) and diverse (low similarity to already-selected).
|