recallnest 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +1 -1
  3. package/README.md +30 -5
  4. package/README_CN.md +30 -5
  5. package/config.json +55 -0
  6. package/connectors/examples/email-adapter.ts +73 -0
  7. package/connectors/examples/logs-adapter.ts +103 -0
  8. package/connectors/examples/rss-adapter.ts +95 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +60 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/ingest-2026-04-13.log +87 -0
  14. package/logs/launchagent-stderr.log +0 -0
  15. package/logs/launchagent-stdout.log +0 -0
  16. package/package.json +1 -1
  17. package/src/cli.ts +30 -0
  18. package/src/connector-types.ts +57 -0
  19. package/src/conversation-importer.ts +25 -1
  20. package/src/data-checkup.ts +31 -1
  21. package/src/doctor.ts +29 -0
  22. package/src/ingest.ts +203 -0
  23. package/src/mcp-server.ts +2 -2
  24. package/src/obsidian-connector.ts +282 -0
  25. package/src/session-schema.ts +2 -2
  26. package/src/source-heartbeat.ts +108 -0
  27. package/.claude-plugin/CLAUDE.md +0 -29
  28. package/.claude-plugin/marketplace.json +0 -18
  29. package/.dockerignore +0 -11
  30. package/.github/workflows/ci.yml +0 -28
  31. package/.mcp.json +0 -9
  32. package/CHANGELOG.md +0 -51
  33. package/DEMO.md +0 -94
  34. package/Dockerfile +0 -20
  35. package/EVAL.md +0 -89
  36. package/FAILURES.md +0 -75
  37. package/OPERATIONS.md +0 -59
  38. package/ROADMAP.md +0 -245
  39. package/SKILL.md +0 -250
  40. package/assets/dashboard.png +0 -0
  41. package/assets/knowledge-graph.png +0 -0
  42. package/assets/recallnest-workbench.png +0 -0
  43. package/assets/screenshots/ui-full.png +0 -0
  44. package/assets/screenshots/ui-overview.png +0 -0
  45. package/assets/screenshots/ui-search.png +0 -0
  46. package/assets/ui/app.js +0 -1031
  47. package/assets/ui/index.html +0 -224
  48. package/assets/ui/styles.css +0 -781
  49. package/assets/wechat-qr.jpg +0 -0
  50. package/bun.lock +0 -337
  51. package/daily-reflections/2026-03-05.md +0 -15
  52. package/daily-reflections/2026-03-06.md +0 -15
  53. package/daily-reflections/2026-03-07.md +0 -7
  54. package/daily-reflections/2026-03-08.md +0 -34
  55. package/daily-reflections/2026-03-09.md +0 -35
  56. package/daily-reflections/2026-03-10.md +0 -36
  57. package/daily-reflections/2026-03-11.md +0 -35
  58. package/daily-reflections/2026-03-12.md +0 -36
  59. package/daily-reflections/2026-03-13.md +0 -35
  60. package/daily-reflections/2026-03-14.md +0 -35
  61. package/daily-reflections/2026-03-15.md +0 -35
  62. package/daily-reflections/2026-03-16.md +0 -41
  63. package/daily-reflections/2026-03-17.md +0 -10
  64. package/daily-reflections/2026-03-18.md +0 -10
  65. package/daily-reflections/2026-03-19.md +0 -10
  66. package/daily-reflections/2026-03-20.md +0 -10
  67. package/daily-reflections/2026-03-21.md +0 -10
  68. package/daily-reflections/2026-03-22.md +0 -10
  69. package/daily-reflections/2026-03-23.md +0 -10
  70. package/daily-reflections/2026-03-24.md +0 -10
  71. package/daily-reflections/2026-03-25.md +0 -32
  72. package/daily-reflections/2026-03-26.md +0 -37
  73. package/daily-reflections/2026-03-27.md +0 -35
  74. package/daily-reflections/2026-03-28.md +0 -35
  75. package/daily-reflections/2026-03-29.md +0 -35
  76. package/daily-reflections/2026-03-30.md +0 -36
  77. package/daily-reflections/2026-03-31.md +0 -35
  78. package/daily-reflections/2026-04-01.md +0 -35
  79. package/daily-reflections/2026-04-02.md +0 -35
  80. package/daily-reflections/2026-04-03.md +0 -37
  81. package/daily-reflections/2026-04-04.md +0 -37
  82. package/daily-reflections/2026-04-05.md +0 -35
  83. package/daily-reflections/2026-04-06.md +0 -36
  84. package/daily-reflections/2026-04-07.md +0 -35
  85. package/daily-reflections/2026-04-08.md +0 -35
  86. package/daily-reflections/2026-04-12.md +0 -35
  87. package/data/alias-map.example.json +0 -22
  88. package/docs/api-reference.md +0 -1291
  89. package/docs/architecture.md +0 -168
  90. package/docs/memory-boundary-contract.md +0 -226
  91. package/docs/memory-categories.md +0 -119
  92. package/docs/pipeline-audit-ultramemory.md +0 -177
  93. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  94. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  95. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  96. package/docs/plans/mcp-tiering-plan.md +0 -126
  97. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  98. package/docs/self-evolution.md +0 -198
  99. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  100. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  101. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  102. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  103. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  104. package/eval/continuity/results-2026-03-16.md +0 -117
  105. package/eval/continuity/results-2026-03-19.md +0 -453
  106. package/eval/continuity/runbook.md +0 -110
  107. package/eval/continuity/seed-cases.md +0 -54
  108. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  109. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  110. package/eval/reports/2026-03-06-baseline.md +0 -82
  111. package/eval/reports/query-expansion.md +0 -138
  112. package/glama.json +0 -6
  113. package/lm +0 -19
  114. package/plan-philosophy-upgrades.md +0 -363
  115. package/plan-product-upgrades.md +0 -155
  116. package/plan-v1.2-distributable.md +0 -105
  117. package/plan.md +0 -140
  118. package/research-memory-evolution.md +0 -440
  119. package/research-rl-agent-memory.md +0 -330
  120. package/scripts/archive-distilled-junk.ts +0 -116
  121. package/scripts/backfill-anchors.ts +0 -130
  122. package/scripts/backfill-l0l1l2.ts +0 -98
  123. package/scripts/blind-scan.ts +0 -588
  124. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  125. package/scripts/cleanup-step2-template-dups.ts +0 -113
  126. package/scripts/cleanup-step3-stale-content.ts +0 -149
  127. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  128. package/scripts/delete-distilled-junk.ts +0 -88
  129. package/scripts/diagnose-vector-search.ts +0 -132
  130. package/scripts/distill-facts.ts +0 -305
  131. package/scripts/health-check.ts +0 -343
  132. package/scripts/incremental-ingest.sh +0 -71
  133. package/scripts/migrate-language.ts +0 -75
  134. package/scripts/mlp-health-check.ts +0 -382
  135. package/scripts/pull-claude-desktop.ts +0 -515
  136. package/scripts/re-ingest-cc.sh +0 -66
  137. package/scripts/rebalance-memory-health.ts +0 -181
  138. package/scripts/run-desktop-ingest.sh +0 -11
  139. package/scripts/start-server.sh +0 -12
  140. package/scripts/verify-emotion-decay.ts +0 -148
  141. package/scripts/viz-dashboard.html +0 -423
  142. package/scripts/viz-export.ts +0 -703
  143. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  144. package/src/__tests__/activity-counter.test.ts +0 -96
  145. package/src/__tests__/admission-control.test.ts +0 -206
  146. package/src/__tests__/anchor-generator.test.ts +0 -67
  147. package/src/__tests__/auto-consolidation.test.ts +0 -168
  148. package/src/__tests__/auto-recall.test.ts +0 -131
  149. package/src/__tests__/babel-integration.test.ts +0 -139
  150. package/src/__tests__/batch-dedup.test.ts +0 -65
  151. package/src/__tests__/capture-engine.test.ts +0 -1144
  152. package/src/__tests__/capture-heuristic.test.ts +0 -171
  153. package/src/__tests__/cascade-forget.test.ts +0 -210
  154. package/src/__tests__/category-min-scores.test.ts +0 -33
  155. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  156. package/src/__tests__/circuit-breaker.test.ts +0 -99
  157. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  158. package/src/__tests__/confidence-metadata.test.ts +0 -125
  159. package/src/__tests__/confidence-tracker.test.ts +0 -249
  160. package/src/__tests__/conflict-advisor.test.ts +0 -206
  161. package/src/__tests__/conflict-escalation.test.ts +0 -149
  162. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  163. package/src/__tests__/conflict-output.test.ts +0 -85
  164. package/src/__tests__/conflict-store.test.ts +0 -374
  165. package/src/__tests__/consolidation-engine.test.ts +0 -181
  166. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  167. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  168. package/src/__tests__/context-composer-pins.test.ts +0 -64
  169. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  170. package/src/__tests__/context-reconstructor.test.ts +0 -65
  171. package/src/__tests__/context-renderer.test.ts +0 -91
  172. package/src/__tests__/continuity-eval.test.ts +0 -275
  173. package/src/__tests__/continuity-schema.test.ts +0 -97
  174. package/src/__tests__/conversation-importer.test.ts +0 -692
  175. package/src/__tests__/core-recall-eval.test.ts +0 -235
  176. package/src/__tests__/core-summary.test.ts +0 -109
  177. package/src/__tests__/data-checkup.test.ts +0 -225
  178. package/src/__tests__/decay-exemption.test.ts +0 -84
  179. package/src/__tests__/dedup-actions.test.ts +0 -58
  180. package/src/__tests__/dedup-guard.test.ts +0 -39
  181. package/src/__tests__/deterministic-id.test.ts +0 -39
  182. package/src/__tests__/distill-lock.test.ts +0 -176
  183. package/src/__tests__/doctor.test.ts +0 -303
  184. package/src/__tests__/dream-pipeline.test.ts +0 -232
  185. package/src/__tests__/drill-down.test.ts +0 -108
  186. package/src/__tests__/embedder.test.ts +0 -122
  187. package/src/__tests__/emotion-decay.test.ts +0 -46
  188. package/src/__tests__/emotion-detector.test.ts +0 -56
  189. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  190. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  191. package/src/__tests__/entity-resolver.test.ts +0 -58
  192. package/src/__tests__/ethics-layer.test.ts +0 -476
  193. package/src/__tests__/event-segmenter.test.ts +0 -153
  194. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  195. package/src/__tests__/frequency-tracker.test.ts +0 -113
  196. package/src/__tests__/graph-export.test.ts +0 -489
  197. package/src/__tests__/hotness-blend.test.ts +0 -76
  198. package/src/__tests__/ingest.test.ts +0 -360
  199. package/src/__tests__/interference-detector.test.ts +0 -203
  200. package/src/__tests__/kg-extractor.test.ts +0 -184
  201. package/src/__tests__/kg-store.test.ts +0 -198
  202. package/src/__tests__/llm-consolidation.test.ts +0 -226
  203. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  204. package/src/__tests__/memory-boundaries.test.ts +0 -237
  205. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  206. package/src/__tests__/memory-evolution.test.ts +0 -271
  207. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  208. package/src/__tests__/memory-lint.test.ts +0 -271
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/narrative-architecture.test.ts +0 -456
  212. package/src/__tests__/pending-review.test.ts +0 -77
  213. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  214. package/src/__tests__/ppr-traversal.test.ts +0 -256
  215. package/src/__tests__/prediction-engine.test.ts +0 -349
  216. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  217. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  218. package/src/__tests__/preference-matcher.test.ts +0 -230
  219. package/src/__tests__/preference-slots.test.ts +0 -205
  220. package/src/__tests__/prospective-memory.test.ts +0 -283
  221. package/src/__tests__/query-entity-detector.test.ts +0 -171
  222. package/src/__tests__/query-expander-alias.test.ts +0 -89
  223. package/src/__tests__/query-expander.test.ts +0 -34
  224. package/src/__tests__/recall-governor.test.ts +0 -227
  225. package/src/__tests__/recall-mode.test.ts +0 -64
  226. package/src/__tests__/result-synthesizer.test.ts +0 -198
  227. package/src/__tests__/resume-context.test.ts +0 -4898
  228. package/src/__tests__/retrieval-trace.test.ts +0 -213
  229. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  230. package/src/__tests__/retroactive-boost.test.ts +0 -223
  231. package/src/__tests__/salience-scoring.test.ts +0 -106
  232. package/src/__tests__/scope-inventory.test.ts +0 -200
  233. package/src/__tests__/scope-policy.test.ts +0 -69
  234. package/src/__tests__/session-checkpoint.test.ts +0 -259
  235. package/src/__tests__/session-distiller.test.ts +0 -611
  236. package/src/__tests__/session-gc.test.ts +0 -215
  237. package/src/__tests__/skill-engine.test.ts +0 -546
  238. package/src/__tests__/skill-promotion.test.ts +0 -349
  239. package/src/__tests__/store-scope-audit.test.ts +0 -81
  240. package/src/__tests__/supersede-chain.test.ts +0 -154
  241. package/src/__tests__/temporal-parser.test.ts +0 -241
  242. package/src/__tests__/temporal-validity.test.ts +0 -160
  243. package/src/__tests__/term-registry.test.ts +0 -121
  244. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  245. package/src/__tests__/topic-tag.test.ts +0 -113
  246. package/src/__tests__/trusted-memory.test.ts +0 -828
  247. package/src/__tests__/version-manager.test.ts +0 -213
  248. package/src/__tests__/workflow-observation.test.ts +0 -400
  249. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,181 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
-
3
- import { ConsolidationEngine, DEFAULT_CONSOLIDATION_CONFIG, formatConsolidationResult, type ConsolidationResult } from "../consolidation-engine.js";
4
- import type { MemoryEntry, MemorySearchResult } from "../store.js";
5
-
6
- function makeEntry(overrides: Partial<MemoryEntry> & { id: string; text: string }): MemoryEntry {
7
- return {
8
- vector: [1, 0, 0],
9
- category: "events",
10
- scope: "project:test",
11
- importance: 0.7,
12
- timestamp: Date.now(),
13
- metadata: "{}",
14
- ...overrides,
15
- };
16
- }
17
-
18
- function createMockStore(entries: MemoryEntry[], similarityMap: Map<string, Map<string, number>> = new Map()) {
19
- const data = new Map(entries.map(e => [e.id, { ...e }]));
20
- const updates: Array<{ id: string; metadata: string }> = [];
21
-
22
- return {
23
- updates,
24
- store: {
25
- async list(scopeFilter?: string[], _category?: string, limit = 500, _offset = 0) {
26
- return [...data.values()]
27
- .filter(e => !scopeFilter || scopeFilter.some(s => e.scope === s))
28
- .slice(0, limit);
29
- },
30
- async getById(id: string) {
31
- return data.get(id) ?? null;
32
- },
33
- async vectorSearch(vector: number[], limit = 5, minScore = 0.3, scopeFilter?: string[]) {
34
- // Use the similarity map to compute fake scores
35
- const sourceEntry = [...data.values()].find(e =>
36
- e.vector.length === vector.length && e.vector.every((v, i) => v === vector[i])
37
- );
38
- if (!sourceEntry) return [];
39
-
40
- const sourceMap = similarityMap.get(sourceEntry.id);
41
- if (!sourceMap) return [];
42
-
43
- const results: MemorySearchResult[] = [];
44
- for (const [targetId, score] of sourceMap) {
45
- if (score < minScore) continue;
46
- const target = data.get(targetId);
47
- if (!target) continue;
48
- if (scopeFilter && !scopeFilter.some(s => target.scope === s)) continue;
49
- results.push({ entry: target, score });
50
- }
51
- return results.sort((a, b) => b.score - a.score).slice(0, limit);
52
- },
53
- async update(id: string, upd: { metadata?: string }) {
54
- const entry = data.get(id);
55
- if (!entry) return null;
56
- if (upd.metadata) {
57
- entry.metadata = upd.metadata;
58
- updates.push({ id, metadata: upd.metadata });
59
- }
60
- return entry;
61
- },
62
- },
63
- };
64
- }
65
-
66
- describe("ConsolidationEngine", () => {
67
- it("returns empty result for empty scope", async () => {
68
- const { store } = createMockStore([]);
69
- const engine = new ConsolidationEngine(store);
70
- const result = await engine.run("project:test");
71
- expect(result.originalCount).toBe(0);
72
- expect(result.clustersFound).toBe(0);
73
- });
74
-
75
- it("skips single-entry categories", async () => {
76
- const entries = [makeEntry({ id: "a", text: "only one" })];
77
- const { store } = createMockStore(entries);
78
- const engine = new ConsolidationEngine(store);
79
- const result = await engine.run("project:test");
80
- expect(result.originalCount).toBe(1);
81
- expect(result.clustersFound).toBe(0);
82
- });
83
-
84
- it("merges near-duplicates above mergeThreshold", async () => {
85
- const entryA = makeEntry({ id: "a", text: "I prefer TypeScript", vector: [1, 0, 0], importance: 0.9 });
86
- const entryB = makeEntry({ id: "b", text: "I prefer TypeScript language", vector: [0.99, 0.1, 0], importance: 0.5 });
87
-
88
- const simMap = new Map([
89
- ["a", new Map([["b", 0.95]])],
90
- ["b", new Map([["a", 0.95]])],
91
- ]);
92
-
93
- const { store, updates } = createMockStore([entryA, entryB], simMap);
94
- const engine = new ConsolidationEngine(store, { ...DEFAULT_CONSOLIDATION_CONFIG, mergeThreshold: 0.92 });
95
- const result = await engine.run("project:test");
96
-
97
- expect(result.clustersFound).toBe(1);
98
- expect(result.mergedCount).toBe(1);
99
- // Tier 3.3: Both entries now coexist in a version group instead of archiving.
100
- // Both A and B should have version_group metadata.
101
- const updateA = updates.find(u => u.id === "a");
102
- const updateB = updates.find(u => u.id === "b");
103
- expect(updateA).toBeTruthy();
104
- expect(updateB).toBeTruthy();
105
- const metaA = JSON.parse(updateA!.metadata);
106
- const metaB = JSON.parse(updateB!.metadata);
107
- expect(metaA.version_group).toBeTruthy();
108
- expect(metaB.version_group).toBe(metaA.version_group);
109
- // Canonical (A, higher importance) should have higher rank
110
- expect(metaA.version_rank).toBeGreaterThan(metaB.version_rank);
111
- });
112
-
113
- it("links related entries below mergeThreshold but above clusterThreshold", async () => {
114
- const entryA = makeEntry({ id: "a", text: "TypeScript config", vector: [1, 0, 0] });
115
- const entryB = makeEntry({ id: "b", text: "TypeScript setup", vector: [0.9, 0.1, 0] });
116
-
117
- const simMap = new Map([
118
- ["a", new Map([["b", 0.85]])],
119
- ["b", new Map([["a", 0.85]])],
120
- ]);
121
-
122
- const { store, updates } = createMockStore([entryA, entryB], simMap);
123
- const engine = new ConsolidationEngine(store, { ...DEFAULT_CONSOLIDATION_CONFIG, clusterThreshold: 0.82, mergeThreshold: 0.92 });
124
- const result = await engine.run("project:test");
125
-
126
- expect(result.clustersFound).toBe(1);
127
- expect(result.mergedCount).toBe(0);
128
- expect(result.relationsAdded).toBe(1);
129
- // Both should have clustering metadata
130
- const linkUpdate = updates.find(u => u.id === "b");
131
- expect(linkUpdate).toBeTruthy();
132
- const meta = JSON.parse(linkUpdate!.metadata);
133
- expect(meta.clustered_with).toBe("a");
134
- });
135
-
136
- it("detects heuristic contradictions", async () => {
137
- const entryA = makeEntry({ id: "a", text: "Always use strict mode in TypeScript projects", vector: [1, 0, 0] });
138
- const entryB = makeEntry({ id: "b", text: "Never use strict mode in TypeScript projects", vector: [0.98, 0.1, 0] });
139
-
140
- const simMap = new Map([
141
- ["a", new Map([["b", 0.95]])],
142
- ["b", new Map([["a", 0.95]])],
143
- ]);
144
-
145
- const { store } = createMockStore([entryA, entryB], simMap);
146
- const engine = new ConsolidationEngine(store, DEFAULT_CONSOLIDATION_CONFIG);
147
- const result = await engine.run("project:test");
148
-
149
- expect(result.conflictsDetected.length).toBe(1);
150
- expect(result.conflictsDetected[0].type).toBe("heuristic_contradiction");
151
- });
152
-
153
- it("skips archived entries", async () => {
154
- const entryA = makeEntry({ id: "a", text: "active entry here", vector: [1, 0, 0] });
155
- const entryB = makeEntry({ id: "b", text: "archived entry here", vector: [0.95, 0.1, 0], metadata: JSON.stringify({ evolution: { status: "archived" } }) });
156
-
157
- const { store } = createMockStore([entryA, entryB]);
158
- const engine = new ConsolidationEngine(store);
159
- const result = await engine.run("project:test");
160
-
161
- expect(result.originalCount).toBe(1); // only active
162
- });
163
- });
164
-
165
- describe("formatConsolidationResult", () => {
166
- it("formats a result with conflicts", () => {
167
- const result: ConsolidationResult = {
168
- originalCount: 100,
169
- clustersFound: 5,
170
- mergedCount: 3,
171
- relationsAdded: 7,
172
- conflictsDetected: [{ memoryA: "aaaa-bbbb", memoryB: "cccc-dddd", type: "heuristic_contradiction" }],
173
- scope: "project:test",
174
- };
175
- const text = formatConsolidationResult(result);
176
- expect(text).toContain("Scanned: 100");
177
- expect(text).toContain("Clusters found: 5");
178
- expect(text).toContain("Merged (versioned): 3");
179
- expect(text).toContain("Conflicts:");
180
- });
181
- });
@@ -1,509 +0,0 @@
1
- /**
2
- * Phase 4: Constructive Retrieval — tests for candidate expansion,
3
- * source-map grounding, contradiction detection, and reconstruction pipeline.
4
- */
5
- import { describe, test, expect } from "bun:test";
6
- import {
7
- expandCandidates,
8
- detectContradictions,
9
- computeSourceMapCoverage,
10
- buildPrompt,
11
- reconstruct,
12
- extractCitedIds,
13
- shouldReconstruct,
14
- type CandidateExpansionDeps,
15
- type ReconstructionInput,
16
- type ReconstructionLLMClient,
17
- type ReconstructionOutput,
18
- } from "../context-reconstructor.js";
19
- import type { RetrievalResult } from "../retriever.js";
20
-
21
- // ============================================================================
22
- // Test Helpers
23
- // ============================================================================
24
-
25
- function makeResult(id: string, text: string, importance = 0.7, score = 0.8): RetrievalResult {
26
- return {
27
- entry: {
28
- id,
29
- text,
30
- importance,
31
- category: "entities",
32
- scope: "test",
33
- timestamp: Date.now(),
34
- metadata: "{}",
35
- vector: new Float32Array(0),
36
- },
37
- score,
38
- sources: {},
39
- };
40
- }
41
-
42
- function makeMockLLM(response: string | null): ReconstructionLLMClient {
43
- return {
44
- generateReconstruction: async () => response,
45
- };
46
- }
47
-
48
- // ============================================================================
49
- // expandCandidates
50
- // ============================================================================
51
-
52
- describe("expandCandidates", () => {
53
- test("tags direct results and returns source map", async () => {
54
- const direct = [makeResult("a", "memory A"), makeResult("b", "memory B")];
55
- const { results, sourceMap } = await expandCandidates(direct, {});
56
- expect(results).toHaveLength(2);
57
- expect(sourceMap.get("a")).toEqual({ type: "direct" });
58
- expect(sourceMap.get("b")).toEqual({ type: "direct" });
59
- });
60
-
61
- test("expands via KG neighbors", async () => {
62
- const direct = [makeResult("a", "memory A")];
63
- const deps: CandidateExpansionDeps = {
64
- expandViaKG: async () => [makeResult("kg1", "KG neighbor")],
65
- };
66
- const { results, sourceMap } = await expandCandidates(direct, deps);
67
- expect(results).toHaveLength(2);
68
- expect(sourceMap.get("kg1")).toEqual({ type: "kg_neighbor" });
69
- });
70
-
71
- test("expands via evolution chains", async () => {
72
- const direct = [makeResult("a", "memory A")];
73
- const deps: CandidateExpansionDeps = {
74
- expandViaEvolution: async () => [makeResult("evo1", "evolution successor")],
75
- };
76
- const { results, sourceMap } = await expandCandidates(direct, deps);
77
- expect(results).toHaveLength(2);
78
- expect(sourceMap.get("evo1")).toEqual({ type: "evolution_chain" });
79
- });
80
-
81
- test("expands via clusters", async () => {
82
- const direct = [makeResult("a", "memory A")];
83
- const deps: CandidateExpansionDeps = {
84
- expandViaClusters: async () => [makeResult("cl1", "cluster member")],
85
- };
86
- const { results, sourceMap } = await expandCandidates(direct, deps);
87
- expect(results).toHaveLength(2);
88
- expect(sourceMap.get("cl1")).toEqual({ type: "cluster_member" });
89
- });
90
-
91
- test("expands via narrative siblings", async () => {
92
- const direct = [makeResult("a", "memory A")];
93
- const deps: CandidateExpansionDeps = {
94
- expandViaNarrative: async () => [makeResult("ns1", "narrative sibling")],
95
- };
96
- const { results, sourceMap } = await expandCandidates(direct, deps);
97
- expect(results).toHaveLength(2);
98
- expect(sourceMap.get("ns1")).toEqual({ type: "narrative_sibling" });
99
- });
100
-
101
- test("deduplicates across expansion sources", async () => {
102
- const direct = [makeResult("a", "memory A")];
103
- const deps: CandidateExpansionDeps = {
104
- expandViaKG: async () => [makeResult("shared", "from KG")],
105
- expandViaEvolution: async () => [makeResult("shared", "from evo")],
106
- };
107
- const { results, sourceMap } = await expandCandidates(direct, deps);
108
- expect(results).toHaveLength(2);
109
- // First expansion wins
110
- expect(sourceMap.get("shared")).toEqual({ type: "kg_neighbor" });
111
- });
112
-
113
- test("respects EXPANSION_CAP of 20", async () => {
114
- const direct = Array.from({ length: 15 }, (_, i) => makeResult(`d${i}`, `direct ${i}`));
115
- const deps: CandidateExpansionDeps = {
116
- expandViaKG: async () => Array.from({ length: 10 }, (_, i) => makeResult(`kg${i}`, `kg ${i}`)),
117
- };
118
- const { results } = await expandCandidates(direct, deps);
119
- expect(results.length).toBeLessThanOrEqual(20);
120
- });
121
-
122
- test("handles expansion failures gracefully", async () => {
123
- const direct = [makeResult("a", "memory A")];
124
- const deps: CandidateExpansionDeps = {
125
- expandViaKG: async () => { throw new Error("KG unavailable"); },
126
- expandViaEvolution: async () => [makeResult("evo1", "evolution ok")],
127
- };
128
- const { results } = await expandCandidates(direct, deps);
129
- expect(results).toHaveLength(2); // direct + evolution (KG failed silently)
130
- });
131
-
132
- test("no expansion deps returns only direct results", async () => {
133
- const direct = [makeResult("a", "A"), makeResult("b", "B")];
134
- const { results, sourceMap } = await expandCandidates(direct, {});
135
- expect(results).toHaveLength(2);
136
- expect(sourceMap.size).toBe(2);
137
- });
138
-
139
- test("parallel expansion sources all contribute", async () => {
140
- const direct = [makeResult("a", "memory A")];
141
- const deps: CandidateExpansionDeps = {
142
- expandViaKG: async () => [makeResult("kg1", "KG")],
143
- expandViaEvolution: async () => [makeResult("evo1", "evo")],
144
- expandViaClusters: async () => [makeResult("cl1", "cluster")],
145
- expandViaNarrative: async () => [makeResult("ns1", "narrative")],
146
- };
147
- const { results, sourceMap } = await expandCandidates(direct, deps);
148
- expect(results).toHaveLength(5);
149
- expect(sourceMap.get("kg1")?.type).toBe("kg_neighbor");
150
- expect(sourceMap.get("evo1")?.type).toBe("evolution_chain");
151
- expect(sourceMap.get("cl1")?.type).toBe("cluster_member");
152
- expect(sourceMap.get("ns1")?.type).toBe("narrative_sibling");
153
- });
154
- });
155
-
156
- // ============================================================================
157
- // detectContradictions
158
- // ============================================================================
159
-
160
- describe("detectContradictions", () => {
161
- test("detects negation-pattern contradiction", () => {
162
- const results = [
163
- makeResult("a", "You should always use TypeScript for projects"),
164
- makeResult("b", "You should not always use TypeScript for every project"),
165
- ];
166
- const contradictions = detectContradictions(results);
167
- expect(contradictions.length).toBeGreaterThanOrEqual(1);
168
- expect(contradictions[0].memoryIds).toEqual(["a", "b"]);
169
- });
170
-
171
- test("no contradictions for compatible memories", () => {
172
- const results = [
173
- makeResult("a", "User prefers dark mode"),
174
- makeResult("b", "User uses Bun runtime"),
175
- ];
176
- expect(detectContradictions(results)).toHaveLength(0);
177
- });
178
-
179
- test("caps at 5 contradictions", () => {
180
- // Create many contradicting pairs
181
- const results: RetrievalResult[] = [];
182
- for (let i = 0; i < 12; i++) {
183
- const text = i % 2 === 0
184
- ? `Feature ${Math.floor(i / 2)} must always be enabled for deployment`
185
- : `Feature ${Math.floor(i / 2)} should never be enabled for deployment`;
186
- results.push(makeResult(`m${i}`, text));
187
- }
188
- const contradictions = detectContradictions(results);
189
- expect(contradictions.length).toBeLessThanOrEqual(5);
190
- });
191
-
192
- test("handles empty results", () => {
193
- expect(detectContradictions([])).toHaveLength(0);
194
- });
195
-
196
- test("handles single result", () => {
197
- expect(detectContradictions([makeResult("a", "solo")])).toHaveLength(0);
198
- });
199
- });
200
-
201
- // ============================================================================
202
- // computeSourceMapCoverage
203
- // ============================================================================
204
-
205
- describe("computeSourceMapCoverage", () => {
206
- test("full coverage when all sentences have valid citations", () => {
207
- const text = "User works on RecallNest [src:a]. They use Bun runtime [src:b].";
208
- const validIds = new Set(["a", "b"]);
209
- expect(computeSourceMapCoverage(text, validIds)).toBe(1.0);
210
- });
211
-
212
- test("partial coverage when some sentences lack citations", () => {
213
- const text = "User works on RecallNest [src:a]. No citation here. Another fact [src:b].";
214
- const validIds = new Set(["a", "b"]);
215
- const coverage = computeSourceMapCoverage(text, validIds);
216
- expect(coverage).toBeGreaterThan(0.5);
217
- expect(coverage).toBeLessThan(1.0);
218
- });
219
-
220
- test("zero coverage when no valid citations", () => {
221
- const text = "Some text [src:invalid]. More text [src:also_invalid].";
222
- const validIds = new Set(["real_id"]);
223
- expect(computeSourceMapCoverage(text, validIds)).toBe(0);
224
- });
225
-
226
- test("handles empty text", () => {
227
- expect(computeSourceMapCoverage("", new Set(["a"]))).toBe(0);
228
- });
229
-
230
- test("filters out short sentences (<=5 chars)", () => {
231
- const text = "OK. User works on RecallNest [src:a].";
232
- const coverage = computeSourceMapCoverage(text, new Set(["a"]));
233
- expect(coverage).toBe(1.0); // "OK." is filtered, only the cited sentence counts
234
- });
235
- });
236
-
237
- // ============================================================================
238
- // buildPrompt (checkpoint context injection)
239
- // ============================================================================
240
-
241
- describe("buildPrompt with checkpoint context", () => {
242
- const baseInput: ReconstructionInput = {
243
- query: "test query",
244
- results: [makeResult("a", "memory A"), makeResult("b", "memory B"), makeResult("c", "memory C")],
245
- mode: "resume",
246
- };
247
-
248
- test("includes open loops in system prompt", () => {
249
- const input: ReconstructionInput = {
250
- ...baseInput,
251
- checkpointContext: {
252
- openLoops: ["Fix CI pipeline", "Review PR #42"],
253
- },
254
- };
255
- const { system } = buildPrompt(input);
256
- expect(system).toContain("Open loops");
257
- expect(system).toContain("Fix CI pipeline");
258
- expect(system).toContain("Review PR #42");
259
- });
260
-
261
- test("includes next actions in system prompt", () => {
262
- const input: ReconstructionInput = {
263
- ...baseInput,
264
- checkpointContext: {
265
- nextActions: ["Deploy to staging", "Run integration tests"],
266
- },
267
- };
268
- const { system } = buildPrompt(input);
269
- expect(system).toContain("Next actions");
270
- expect(system).toContain("Deploy to staging");
271
- });
272
-
273
- test("includes scope in system prompt", () => {
274
- const input: ReconstructionInput = {
275
- ...baseInput,
276
- checkpointContext: { scope: "project:recallnest" },
277
- };
278
- const { system } = buildPrompt(input);
279
- expect(system).toContain("Active scope: project:recallnest");
280
- });
281
-
282
- test("no checkpoint context = no checkpoint hint", () => {
283
- const { system } = buildPrompt(baseInput);
284
- expect(system).not.toContain("Checkpoint state");
285
- });
286
-
287
- test("empty checkpoint context = no checkpoint hint", () => {
288
- const input: ReconstructionInput = {
289
- ...baseInput,
290
- checkpointContext: {},
291
- };
292
- const { system } = buildPrompt(input);
293
- expect(system).not.toContain("Checkpoint state");
294
- });
295
- });
296
-
297
- // ============================================================================
298
- // reconstruct (full pipeline)
299
- // ============================================================================
300
-
301
- describe("reconstruct pipeline", () => {
302
- const baseResults = [
303
- makeResult("m1", "User prefers TypeScript"),
304
- makeResult("m2", "Project uses Bun runtime"),
305
- makeResult("m3", "Deploy target is Docker"),
306
- ];
307
-
308
- test("returns reconstructed text with typed sources", async () => {
309
- const llm = makeMockLLM(
310
- "The user prefers TypeScript [src:m1]. The project uses Bun [src:m2]. Deployed via Docker [src:m3]."
311
- );
312
- const result = await reconstruct(
313
- { query: "project setup", results: baseResults, mode: "search" },
314
- llm,
315
- );
316
- expect(result.reconstructed).toBeTruthy();
317
- expect(result.sources.length).toBeGreaterThanOrEqual(1);
318
- // Sources have typed structure
319
- for (const s of result.sources) {
320
- expect(s).toHaveProperty("id");
321
- expect(s).toHaveProperty("source");
322
- expect(s.source).toHaveProperty("type");
323
- expect(s).toHaveProperty("contribution");
324
- }
325
- });
326
-
327
- test("returns contradictions array (even when empty)", async () => {
328
- const llm = makeMockLLM("Summary [src:m1]. Info [src:m2]. Details [src:m3].");
329
- const result = await reconstruct(
330
- { query: "test", results: baseResults, mode: "search" },
331
- llm,
332
- );
333
- expect(result).toHaveProperty("contradictions");
334
- expect(Array.isArray(result.contradictions)).toBe(true);
335
- });
336
-
337
- test("returns coverage score", async () => {
338
- const llm = makeMockLLM("Cited fact [src:m1]. Another fact [src:m2]. Third fact [src:m3].");
339
- const result = await reconstruct(
340
- { query: "test", results: baseResults, mode: "search" },
341
- llm,
342
- );
343
- expect(result).toHaveProperty("coverage");
344
- expect(typeof result.coverage).toBe("number");
345
- });
346
-
347
- test("fallback on LLM empty response", async () => {
348
- const llm = makeMockLLM(null);
349
- const result = await reconstruct(
350
- { query: "test", results: baseResults, mode: "search" },
351
- llm,
352
- );
353
- expect(result.reconstructed).toBeNull();
354
- expect(result.fallbackReason).toBe("llm_empty");
355
- expect(result.sources).toHaveLength(0);
356
- expect(result.contradictions).toHaveLength(0);
357
- });
358
-
359
- test("fallback on low grounding coverage", async () => {
360
- // Response with no valid citations
361
- const llm = makeMockLLM("Some text without any citations. Another ungrounded sentence.");
362
- const result = await reconstruct(
363
- { query: "test", results: baseResults, mode: "search" },
364
- llm,
365
- );
366
- expect(result.reconstructed).toBeNull();
367
- expect(result.fallbackReason).toBe("low_grounding");
368
- });
369
-
370
- test("removes sentences with invalid IDs and lowers confidence", async () => {
371
- const llm = makeMockLLM(
372
- "Valid fact [src:m1]. Fake source [src:FAKE]. Another valid [src:m2]. Third valid [src:m3]."
373
- );
374
- const result = await reconstruct(
375
- { query: "test", results: baseResults, mode: "search" },
376
- llm,
377
- );
378
- if (result.reconstructed) {
379
- expect(result.reconstructed).not.toContain("[src:FAKE]");
380
- expect(result.confidence).toBeLessThan(1.0);
381
- }
382
- });
383
-
384
- test("respects 3s timeout", async () => {
385
- const slowLLM: ReconstructionLLMClient = {
386
- generateReconstruction: async () => {
387
- await new Promise(r => setTimeout(r, 4000));
388
- return "too late [src:m1]";
389
- },
390
- };
391
- const result = await reconstruct(
392
- { query: "test", results: baseResults, mode: "search" },
393
- slowLLM,
394
- );
395
- expect(result.fallbackReason).toBe("timeout");
396
- }, 6000);
397
-
398
- test("with expansion deps — expanded candidates used in reconstruction", async () => {
399
- const expanded = makeResult("exp1", "expanded info");
400
- const deps: CandidateExpansionDeps = {
401
- expandViaKG: async () => [expanded],
402
- };
403
- const llm = makeMockLLM(
404
- "Main fact [src:m1]. KG expanded [src:exp1]. More info [src:m2]. Detail [src:m3]."
405
- );
406
- const result = await reconstruct(
407
- { query: "test", results: baseResults, mode: "search" },
408
- llm,
409
- deps,
410
- );
411
- expect(result.reconstructed).toBeTruthy();
412
- // Check that expanded source is tracked as kg_neighbor
413
- const expSource = result.sources.find(s => s.id === "exp1");
414
- expect(expSource).toBeTruthy();
415
- expect(expSource!.source.type).toBe("kg_neighbor");
416
- });
417
-
418
- test("source contributions extracted from cited sentences", async () => {
419
- const llm = makeMockLLM(
420
- "The user loves TypeScript and uses it everywhere [src:m1]. Bun is the runtime [src:m2]. Docker deploys [src:m3]."
421
- );
422
- const result = await reconstruct(
423
- { query: "test", results: baseResults, mode: "search" },
424
- llm,
425
- );
426
- if (result.sources.length > 0) {
427
- const m1Source = result.sources.find(s => s.id === "m1");
428
- expect(m1Source?.contribution).toBeTruthy();
429
- expect(m1Source?.contribution).toContain("TypeScript");
430
- }
431
- });
432
-
433
- test("checkpoint context passed through to LLM", async () => {
434
- let capturedSystem = "";
435
- const spy: ReconstructionLLMClient = {
436
- generateReconstruction: async (system, _user) => {
437
- capturedSystem = system;
438
- return "Fact [src:m1]. Detail [src:m2]. Info [src:m3].";
439
- },
440
- };
441
- await reconstruct(
442
- {
443
- query: "test",
444
- results: baseResults,
445
- mode: "resume",
446
- checkpointContext: {
447
- openLoops: ["Fix bug #99"],
448
- scope: "project:test",
449
- },
450
- },
451
- spy,
452
- );
453
- expect(capturedSystem).toContain("Fix bug #99");
454
- expect(capturedSystem).toContain("project:test");
455
- });
456
- });
457
-
458
- // ============================================================================
459
- // Integration: ReconstructionOutput shape contract
460
- // ============================================================================
461
-
462
- describe("ReconstructionOutput shape", () => {
463
- test("successful reconstruction has all required fields", async () => {
464
- const results = [
465
- makeResult("a", "fact A"),
466
- makeResult("b", "fact B"),
467
- makeResult("c", "fact C"),
468
- ];
469
- const llm = makeMockLLM("Summary of A [src:a]. Summary of B [src:b]. Summary of C [src:c].");
470
- const output = await reconstruct(
471
- { query: "test", results, mode: "search" },
472
- llm,
473
- );
474
-
475
- // Shape validation
476
- expect(output).toHaveProperty("reconstructed");
477
- expect(output).toHaveProperty("sources");
478
- expect(output).toHaveProperty("confidence");
479
- expect(output).toHaveProperty("contradictions");
480
- expect(output).toHaveProperty("coverage");
481
- expect(output).toHaveProperty("raw");
482
-
483
- // Type validation
484
- expect(typeof output.confidence).toBe("number");
485
- expect(typeof output.coverage).toBe("number");
486
- expect(Array.isArray(output.sources)).toBe(true);
487
- expect(Array.isArray(output.contradictions)).toBe(true);
488
- expect(Array.isArray(output.raw)).toBe(true);
489
- });
490
-
491
- test("failed reconstruction has all required fields", async () => {
492
- const results = [
493
- makeResult("a", "fact A"),
494
- makeResult("b", "fact B"),
495
- makeResult("c", "fact C"),
496
- ];
497
- const llm = makeMockLLM(null);
498
- const output = await reconstruct(
499
- { query: "test", results, mode: "search" },
500
- llm,
501
- );
502
-
503
- expect(output.reconstructed).toBeNull();
504
- expect(output.sources).toHaveLength(0);
505
- expect(output.contradictions).toHaveLength(0);
506
- expect(output.coverage).toBe(0);
507
- expect(output.fallbackReason).toBeTruthy();
508
- });
509
- });