recallnest 1.4.1 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +7 -3
  3. package/README.md +202 -130
  4. package/README_CN.md +263 -190
  5. package/bin/recallnest.mjs +62 -0
  6. package/config.json +55 -0
  7. package/daily-recall.sh +23 -0
  8. package/integrations/setup-all.sh +31 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +30 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/launchagent-stderr.log +0 -0
  14. package/logs/launchagent-stdout.log +0 -0
  15. package/package.json +10 -3
  16. package/src/access-tracker.ts +3 -2
  17. package/src/activity-counter.ts +109 -0
  18. package/src/adaptive-retrieval.ts +6 -5
  19. package/src/admission-control.ts +163 -0
  20. package/src/anchor-generator.ts +80 -0
  21. package/src/api-server.ts +10 -0
  22. package/src/audit-log.ts +122 -0
  23. package/src/auto-gc.ts +98 -29
  24. package/src/auto-recall.ts +33 -4
  25. package/src/capture-engine.ts +372 -23
  26. package/src/cascade-forget.ts +3 -2
  27. package/src/cli.ts +107 -5
  28. package/src/compat.ts +22 -0
  29. package/src/confidence-tracker.ts +210 -47
  30. package/src/conflict-store.ts +12 -3
  31. package/src/consolidation-engine.ts +296 -8
  32. package/src/context-collapse-renderer.ts +222 -0
  33. package/src/context-composer-stable-selection.ts +5 -1
  34. package/src/context-composer.ts +239 -4
  35. package/src/context-reconstructor.ts +404 -0
  36. package/src/conversation-importer.ts +441 -0
  37. package/src/data-checkup.ts +232 -0
  38. package/src/decay-engine.ts +84 -0
  39. package/src/distill-lock.ts +122 -0
  40. package/src/doctor.ts +8 -10
  41. package/src/dream-pipeline.ts +227 -0
  42. package/src/emotion-detector.ts +90 -0
  43. package/src/eval.ts +3 -2
  44. package/src/forget-engine.ts +269 -0
  45. package/src/frequency-tracker.ts +175 -0
  46. package/src/graph-export.ts +644 -0
  47. package/src/ingest.ts +36 -6
  48. package/src/interference-detector.ts +187 -0
  49. package/src/kg-extractor.ts +15 -24
  50. package/src/kg-store.ts +2 -1
  51. package/src/language-hook.ts +67 -0
  52. package/src/llm-client.ts +289 -6
  53. package/src/mcp-server.ts +707 -80
  54. package/src/memory-assets.ts +22 -5
  55. package/src/memory-evolution.ts +365 -0
  56. package/src/memory-lint.ts +429 -0
  57. package/src/memory-output.ts +69 -0
  58. package/src/memory-schema.ts +74 -2
  59. package/src/narrative-schema.ts +79 -0
  60. package/src/narrative-tagger.ts +164 -0
  61. package/src/pii-detector.ts +98 -0
  62. package/src/post-retrieval-filter.ts +76 -0
  63. package/src/prediction-engine.ts +253 -0
  64. package/src/preference-slots.ts +65 -2
  65. package/src/prospective-memory.ts +215 -0
  66. package/src/query-expander.ts +67 -0
  67. package/src/recall-governor.ts +183 -0
  68. package/src/retention-policy.ts +90 -0
  69. package/src/retriever.ts +828 -55
  70. package/src/rif.ts +34 -8
  71. package/src/runtime-config.ts +23 -8
  72. package/src/scope-inventory.ts +4 -3
  73. package/src/scope-policy.ts +4 -0
  74. package/src/session-distiller.ts +307 -0
  75. package/src/session-output.ts +40 -0
  76. package/src/session-schema.ts +37 -0
  77. package/src/session-store.ts +7 -2
  78. package/src/skill-engine.ts +208 -0
  79. package/src/skill-promotion.ts +293 -0
  80. package/src/skill-schema.ts +39 -0
  81. package/src/store.ts +96 -10
  82. package/src/topic-tag.ts +66 -0
  83. package/src/tracker.ts +3 -1
  84. package/src/ui-server.ts +80 -8
  85. package/src/workflow-observation-scope-review.ts +2 -1
  86. package/src/workflow-observation-store.ts +2 -1
  87. package/src/write-verifier.ts +126 -0
  88. package/.claude-plugin/CLAUDE.md +0 -29
  89. package/.claude-plugin/marketplace.json +0 -18
  90. package/.github/workflows/ci.yml +0 -28
  91. package/.mcp.json +0 -9
  92. package/CHANGELOG.md +0 -51
  93. package/DEMO.md +0 -94
  94. package/EVAL.md +0 -89
  95. package/FAILURES.md +0 -75
  96. package/OPERATIONS.md +0 -59
  97. package/ROADMAP.md +0 -198
  98. package/assets/recallnest-workbench.png +0 -0
  99. package/assets/ui/app.js +0 -807
  100. package/assets/ui/index.html +0 -154
  101. package/assets/ui/styles.css +0 -605
  102. package/assets/wechat-qr.jpg +0 -0
  103. package/daily-reflections/2026-03-05.md +0 -15
  104. package/daily-reflections/2026-03-06.md +0 -15
  105. package/daily-reflections/2026-03-07.md +0 -7
  106. package/daily-reflections/2026-03-08.md +0 -34
  107. package/daily-reflections/2026-03-09.md +0 -35
  108. package/daily-reflections/2026-03-10.md +0 -36
  109. package/daily-reflections/2026-03-11.md +0 -35
  110. package/daily-reflections/2026-03-12.md +0 -36
  111. package/daily-reflections/2026-03-13.md +0 -35
  112. package/daily-reflections/2026-03-14.md +0 -35
  113. package/daily-reflections/2026-03-15.md +0 -35
  114. package/daily-reflections/2026-03-16.md +0 -41
  115. package/daily-reflections/2026-03-17.md +0 -10
  116. package/daily-reflections/2026-03-18.md +0 -10
  117. package/daily-reflections/2026-03-19.md +0 -10
  118. package/daily-reflections/2026-03-20.md +0 -10
  119. package/daily-reflections/2026-03-21.md +0 -10
  120. package/daily-reflections/2026-03-22.md +0 -10
  121. package/daily-reflections/2026-03-23.md +0 -10
  122. package/daily-reflections/2026-03-24.md +0 -10
  123. package/daily-reflections/2026-03-25.md +0 -32
  124. package/daily-reflections/2026-03-26.md +0 -37
  125. package/daily-reflections/2026-03-27.md +0 -35
  126. package/daily-reflections/2026-03-28.md +0 -35
  127. package/daily-reflections/2026-03-29.md +0 -35
  128. package/daily-reflections/2026-03-30.md +0 -36
  129. package/daily-reflections/2026-03-31.md +0 -35
  130. package/daily-reflections/2026-04-01.md +0 -35
  131. package/docs/api-reference.md +0 -1291
  132. package/docs/architecture.md +0 -168
  133. package/docs/memory-boundary-contract.md +0 -226
  134. package/docs/memory-categories.md +0 -119
  135. package/docs/pipeline-audit-ultramemory.md +0 -177
  136. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  137. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  138. package/docs/plans/mcp-tiering-plan.md +0 -126
  139. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  140. package/docs/self-evolution.md +0 -198
  141. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  142. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  143. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  144. package/eval/continuity/results-2026-03-16.md +0 -117
  145. package/eval/continuity/results-2026-03-19.md +0 -453
  146. package/eval/continuity/runbook.md +0 -110
  147. package/eval/continuity/seed-cases.md +0 -54
  148. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  149. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  150. package/eval/reports/2026-03-06-baseline.md +0 -82
  151. package/eval/reports/query-expansion.md +0 -138
  152. package/lm +0 -19
  153. package/plan-v1.2-distributable.md +0 -105
  154. package/scripts/archive-distilled-junk.ts +0 -116
  155. package/scripts/backfill-l0l1l2.ts +0 -98
  156. package/scripts/blind-scan.ts +0 -588
  157. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  158. package/scripts/cleanup-step2-template-dups.ts +0 -113
  159. package/scripts/cleanup-step3-stale-content.ts +0 -149
  160. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  161. package/scripts/delete-distilled-junk.ts +0 -88
  162. package/scripts/distill-facts.ts +0 -305
  163. package/scripts/health-check.ts +0 -343
  164. package/scripts/incremental-ingest.sh +0 -71
  165. package/scripts/mlp-health-check.ts +0 -382
  166. package/scripts/pull-claude-desktop.ts +0 -515
  167. package/scripts/re-ingest-cc.sh +0 -66
  168. package/scripts/rebalance-memory-health.ts +0 -181
  169. package/scripts/run-desktop-ingest.sh +0 -11
  170. package/scripts/start-server.sh +0 -12
  171. package/scripts/viz-dashboard.html +0 -423
  172. package/scripts/viz-export.ts +0 -703
  173. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  174. package/src/__tests__/auto-consolidation.test.ts +0 -168
  175. package/src/__tests__/auto-recall.test.ts +0 -131
  176. package/src/__tests__/batch-dedup.test.ts +0 -65
  177. package/src/__tests__/capture-engine.test.ts +0 -1142
  178. package/src/__tests__/capture-heuristic.test.ts +0 -171
  179. package/src/__tests__/cascade-forget.test.ts +0 -210
  180. package/src/__tests__/category-min-scores.test.ts +0 -33
  181. package/src/__tests__/confidence-tracker.test.ts +0 -196
  182. package/src/__tests__/conflict-advisor.test.ts +0 -206
  183. package/src/__tests__/conflict-escalation.test.ts +0 -149
  184. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  185. package/src/__tests__/conflict-output.test.ts +0 -85
  186. package/src/__tests__/conflict-store.test.ts +0 -374
  187. package/src/__tests__/consolidation-engine.test.ts +0 -181
  188. package/src/__tests__/context-composer-pins.test.ts +0 -64
  189. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  190. package/src/__tests__/context-renderer.test.ts +0 -91
  191. package/src/__tests__/continuity-eval.test.ts +0 -275
  192. package/src/__tests__/continuity-schema.test.ts +0 -97
  193. package/src/__tests__/core-recall-eval.test.ts +0 -235
  194. package/src/__tests__/core-summary.test.ts +0 -109
  195. package/src/__tests__/dedup-actions.test.ts +0 -58
  196. package/src/__tests__/deterministic-id.test.ts +0 -39
  197. package/src/__tests__/doctor.test.ts +0 -303
  198. package/src/__tests__/drill-down.test.ts +0 -108
  199. package/src/__tests__/embedder.test.ts +0 -122
  200. package/src/__tests__/entity-resolver.test.ts +0 -58
  201. package/src/__tests__/event-segmenter.test.ts +0 -153
  202. package/src/__tests__/hotness-blend.test.ts +0 -76
  203. package/src/__tests__/ingest.test.ts +0 -360
  204. package/src/__tests__/kg-extractor.test.ts +0 -184
  205. package/src/__tests__/kg-store.test.ts +0 -198
  206. package/src/__tests__/llm-consolidation.test.ts +0 -226
  207. package/src/__tests__/memory-boundaries.test.ts +0 -237
  208. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/ppr-traversal.test.ts +0 -256
  212. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  213. package/src/__tests__/preference-matcher.test.ts +0 -230
  214. package/src/__tests__/preference-slots.test.ts +0 -106
  215. package/src/__tests__/prospective-memory.test.ts +0 -283
  216. package/src/__tests__/query-entity-detector.test.ts +0 -171
  217. package/src/__tests__/query-expander.test.ts +0 -34
  218. package/src/__tests__/recall-mode.test.ts +0 -64
  219. package/src/__tests__/result-synthesizer.test.ts +0 -198
  220. package/src/__tests__/resume-context.test.ts +0 -4743
  221. package/src/__tests__/retrieval-trace.test.ts +0 -213
  222. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  223. package/src/__tests__/retroactive-boost.test.ts +0 -223
  224. package/src/__tests__/scope-inventory.test.ts +0 -200
  225. package/src/__tests__/scope-policy.test.ts +0 -69
  226. package/src/__tests__/session-checkpoint.test.ts +0 -259
  227. package/src/__tests__/session-gc.test.ts +0 -215
  228. package/src/__tests__/store-scope-audit.test.ts +0 -81
  229. package/src/__tests__/temporal-parser.test.ts +0 -241
  230. package/src/__tests__/term-registry.test.ts +0 -121
  231. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  232. package/src/__tests__/version-manager.test.ts +0 -213
  233. package/src/__tests__/workflow-observation.test.ts +0 -400
@@ -1,305 +0,0 @@
1
- #!/usr/bin/env bun
2
- /**
3
- * Bulk distillation: Convert raw "fact" conversation fragments into
4
- * structured knowledge using smartExtractBatch().
5
- *
6
- * Usage:
7
- * bun scripts/distill-facts.ts --dry-run # Process 1 session, print results
8
- * bun scripts/distill-facts.ts # Batch all sessions with checkpoint
9
- * bun scripts/distill-facts.ts --scope cc:09541b9c # Process specific session only
10
- */
11
-
12
- import path from "path";
13
- import fs from "fs";
14
- import { loadConfig, createComponents } from "../src/runtime-config.js";
15
- import { buildDistillationBoundary } from "../src/memory-boundaries.js";
16
- import type { MemoryEntry } from "../src/store.js";
17
- import type { SmartExtraction, LLMClient } from "../src/llm-client.js";
18
- import type { MemoryStore } from "../src/store.js";
19
- import type { Embedder } from "../src/embedder.js";
20
-
21
- // ─── Config ─────────────────────────────────────────────────────────────────
22
-
23
- const BATCH_SIZE = 20;
24
- const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
25
-
26
- interface DistillProgress {
27
- startedAt: string;
28
- lastUpdated: string;
29
- completedScopes: string[];
30
- stats: {
31
- totalFacts: number;
32
- processedFacts: number;
33
- newRecords: number;
34
- dedupedSkips: number;
35
- errors: number;
36
- };
37
- }
38
-
39
- function loadProgress(): DistillProgress | null {
40
- try {
41
- return JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
42
- } catch {
43
- return null;
44
- }
45
- }
46
-
47
- function saveProgress(progress: DistillProgress): void {
48
- fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
49
- }
50
-
51
- // ─── Core Logic ─────────────────────────────────────────────────────────────
52
-
53
- async function distillBatch(
54
- texts: string[],
55
- originals: MemoryEntry[],
56
- store: MemoryStore,
57
- embedder: Embedder,
58
- llm: LLMClient,
59
- dryRun = false,
60
- ): Promise<{ newCount: number; dedupCount: number; errors: number }> {
61
- let newCount = 0;
62
- let dedupCount = 0;
63
- let errors = 0;
64
-
65
- // Step 1: Extract knowledge via LLM
66
- // smartExtractBatch returns (SmartExtraction | null)[] with same length as input
67
- let rawExtractions: (SmartExtraction | null)[];
68
- try {
69
- rawExtractions = await llm.smartExtractBatch(texts);
70
- } catch (e) {
71
- console.error(` LLM extraction failed: ${e}`);
72
- return { newCount: 0, dedupCount: 0, errors: texts.length };
73
- }
74
-
75
- const validIndices: number[] = [];
76
- for (let i = 0; i < rawExtractions.length; i++) {
77
- const ext = rawExtractions[i];
78
- if (ext && (ext.l0.length > 10 || ext.l1.length > 10)) {
79
- validIndices.push(i);
80
- }
81
- }
82
-
83
- if (validIndices.length === 0) {
84
- return { newCount: 0, dedupCount: texts.length, errors: 0 };
85
- }
86
-
87
- // Step 3: Generate embeddings
88
- const textsToEmbed = validIndices.map(i => {
89
- const ext = rawExtractions[i]!;
90
- return ext.l1 || ext.l0;
91
- });
92
-
93
- let vectors: number[][];
94
- try {
95
- vectors = await embedder.embedBatchPassage(textsToEmbed);
96
- } catch (e) {
97
- console.error(` Embedding failed: ${e}`);
98
- return { newCount: 0, dedupCount: 0, errors: validIndices.length };
99
- }
100
-
101
- // Step 4: Dedup + store each extracted record
102
- const idMapping = new Map<number, string>();
103
- for (let vi = 0; vi < validIndices.length; vi++) {
104
- const idx = validIndices[vi];
105
- const extraction = rawExtractions[idx]!;
106
- const vector = vectors[vi];
107
- const original = originals[idx];
108
-
109
- // Skip per-record dedup — distilled L0/L1 text is unlikely to duplicate existing records,
110
- // and dedupCheck was the main bottleneck (vector search per record). Post-distillation
111
- // health check can detect duplicates if needed.
112
-
113
- if (dryRun) {
114
- console.log(`\n [${extraction.category}] importance=${extraction.importance}`);
115
- console.log(` src: ${original.text.slice(0, 80)}...`);
116
- console.log(` L0: ${extraction.l0}`);
117
- console.log(` L1: ${(extraction.l1 || "").slice(0, 120)}`);
118
- newCount++;
119
- continue;
120
- }
121
-
122
- // Build new record
123
- const boundary = buildDistillationBoundary(extraction.category as any);
124
- const metadata = JSON.stringify({
125
- source: "distillation",
126
- distilledFrom: original.id,
127
- originalScope: original.scope,
128
- l0: extraction.l0,
129
- l1: extraction.l1,
130
- tier: extraction.importance >= 0.8 ? "working" : "peripheral",
131
- boundary,
132
- });
133
-
134
- try {
135
- const entry = await store.store({
136
- text: extraction.l1 || extraction.l0,
137
- vector,
138
- category: extraction.category as any,
139
- scope: original.scope,
140
- importance: extraction.importance,
141
- metadata,
142
- });
143
- idMapping.set(idx, entry.id);
144
- newCount++;
145
- } catch (e) {
146
- console.error(` Store failed: ${e}`);
147
- errors++;
148
- }
149
- }
150
-
151
- // Step 5: Archive originals (skip in dry-run)
152
- if (dryRun) return { newCount, dedupCount, errors };
153
- for (let vi = 0; vi < validIndices.length; vi++) {
154
- const idx = validIndices[vi];
155
- const original = originals[idx];
156
- const distilledId = idMapping.get(idx);
157
- if (!distilledId) continue; // Was deduped or failed
158
- try {
159
- const existingMeta = original.metadata ? JSON.parse(original.metadata) : {};
160
- const updatedMeta = JSON.stringify({
161
- ...existingMeta,
162
- archived: true,
163
- archivedAt: new Date().toISOString(),
164
- distilledTo: [distilledId],
165
- originalCategory: "fact",
166
- });
167
- await store.update(original.id, { metadata: updatedMeta });
168
- } catch (e) {
169
- console.error(` Archive mark failed for ${original.id}: ${e}`);
170
- }
171
- }
172
-
173
- return { newCount, dedupCount, errors };
174
- }
175
-
176
- // ─── Main ───────────────────────────────────────────────────────────────────
177
-
178
- async function main() {
179
- const args = process.argv.slice(2);
180
- const dryRun = args.includes("--dry-run");
181
- const scopeIdx = args.indexOf("--scope");
182
- const scopeArg = args.find(a => a.startsWith("--scope="))?.split("=")[1]
183
- || (scopeIdx >= 0 ? args[scopeIdx + 1] : undefined);
184
-
185
- const config = loadConfig();
186
- const { store, embedder, llm } = createComponents(config);
187
-
188
- if (!llm) {
189
- console.error("LLM not configured. Check config.json llm section.");
190
- process.exit(1);
191
- }
192
-
193
- const llmTest = await llm.test();
194
- if (!llmTest.success) {
195
- console.error(`LLM connection failed: ${llmTest.error}`);
196
- process.exit(1);
197
- }
198
- console.log(`LLM: ${config.llm?.model}`);
199
-
200
- // Fetch all fact records
201
- console.log("\nReading fact records...");
202
- const allFacts: MemoryEntry[] = await store.list(undefined, "fact", 50000, 0);
203
- console.log(`Total: ${allFacts.length} fact records\n`);
204
-
205
- // Filter out already-archived facts
206
- const unarchived = allFacts.filter(f => {
207
- if (!f.metadata) return true;
208
- try { return !JSON.parse(f.metadata).archived; } catch { return true; }
209
- });
210
- console.log(`Unarchived: ${unarchived.length} (already archived: ${allFacts.length - unarchived.length})\n`);
211
-
212
- // Group by scope
213
- const byScope = new Map<string, MemoryEntry[]>();
214
- for (const f of unarchived) {
215
- const arr = byScope.get(f.scope) || [];
216
- arr.push(f);
217
- byScope.set(f.scope, arr);
218
- }
219
- const scopes = [...byScope.keys()].sort((a, b) => byScope.get(b)!.length - byScope.get(a)!.length);
220
- console.log(`${scopes.length} scopes\n`);
221
-
222
- // Load or create progress
223
- let progress = loadProgress();
224
- if (!progress) {
225
- progress = {
226
- startedAt: new Date().toISOString(),
227
- lastUpdated: new Date().toISOString(),
228
- completedScopes: [],
229
- stats: { totalFacts: unarchived.length, processedFacts: 0, newRecords: 0, dedupedSkips: 0, errors: 0 },
230
- };
231
- }
232
-
233
- // Filter scopes
234
- let targetScopes = scopes;
235
- if (scopeArg) {
236
- targetScopes = scopes.filter(s => s === scopeArg);
237
- if (targetScopes.length === 0) {
238
- console.error(`Scope "${scopeArg}" not found`);
239
- process.exit(1);
240
- }
241
- }
242
- if (dryRun) {
243
- targetScopes = [targetScopes[0]];
244
- console.log(`DRY RUN: processing scope "${targetScopes[0]}" (${byScope.get(targetScopes[0])!.length} records)\n`);
245
- }
246
-
247
- // Process scopes — worker pool with immediate checkpoint per scope
248
- const CONCURRENCY = 5;
249
-
250
- // Filter to pending scopes, sort small-first for faster feedback
251
- const pendingScopes = targetScopes.filter(s => !progress.completedScopes.includes(s));
252
- pendingScopes.sort((a, b) => byScope.get(a)!.length - byScope.get(b)!.length);
253
- const skipped = targetScopes.length - pendingScopes.length;
254
- if (skipped > 0) console.log(`Skipping ${skipped} already-completed scopes`);
255
- console.log(`Processing ${pendingScopes.length} scopes (small-first, concurrency=${CONCURRENCY})\n`);
256
-
257
- let completed = 0;
258
- const totalPending = pendingScopes.length;
259
- let scopeQueue = [...pendingScopes];
260
-
261
- async function worker() {
262
- while (scopeQueue.length > 0) {
263
- const scope = scopeQueue.shift()!;
264
- const facts = byScope.get(scope)!;
265
- let scopeNew = 0, scopeErrors = 0;
266
-
267
- for (let i = 0; i < facts.length; i += BATCH_SIZE) {
268
- const batch = facts.slice(i, i + BATCH_SIZE);
269
- const texts = batch.map(f => f.text);
270
- const result = await distillBatch(texts, batch, store, embedder, llm, dryRun);
271
- scopeNew += result.newCount;
272
- scopeErrors += result.errors;
273
- }
274
-
275
- // Checkpoint immediately
276
- progress.completedScopes.push(scope);
277
- progress.stats.processedFacts += facts.length;
278
- progress.stats.newRecords += scopeNew;
279
- progress.stats.errors += scopeErrors;
280
- progress.lastUpdated = new Date().toISOString();
281
- completed++;
282
-
283
- if (!dryRun) saveProgress(progress);
284
- console.log(`[${completed}/${totalPending}] ${scope} (${facts.length} facts, +${scopeNew} new, ${scopeErrors} err)`);
285
- }
286
- }
287
-
288
- // Launch workers
289
- await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
290
-
291
- // Final summary
292
- console.log("\n" + "=".repeat(60));
293
- console.log("Distillation summary:");
294
- console.log(` Total facts: ${progress.stats.totalFacts}`);
295
- console.log(` Processed: ${progress.stats.processedFacts}`);
296
- console.log(` New knowledge: ${progress.stats.newRecords}`);
297
- console.log(` Dedup skipped: ${progress.stats.dedupedSkips}`);
298
- console.log(` Errors: ${progress.stats.errors}`);
299
- console.log(` Compression: ${(progress.stats.processedFacts / Math.max(progress.stats.newRecords, 1)).toFixed(1)}:1`);
300
- if (dryRun) {
301
- console.log("\nDRY RUN — no data was written. Remove --dry-run to run for real.");
302
- }
303
- }
304
-
305
- main().catch(console.error);
@@ -1,343 +0,0 @@
1
- /**
2
- * RecallNest 记忆体检脚本
3
- * 只读分析 LanceDB 数据,不写入任何内容
4
- *
5
- * 检查项:
6
- * 1. Category 分布(含百分比)
7
- * 2. Tier 分布(peripheral/working/core)
8
- * 3. 死记忆(accessCount = 0 或无 accessCount)
9
- * 4. Importance 分布
10
- * 5. 来源分布(CC/Codex/Gemini/manual)
11
- * 6. 年龄分布
12
- * 7. 近似重复检测(采样 cosine similarity)
13
- */
14
-
15
- import lancedb from "@lancedb/lancedb";
16
- import path from "path";
17
-
18
- const DB_PATH = path.join(import.meta.dir, "..", "data", "lancedb");
19
- const TABLE_NAME = "memories";
20
-
21
- interface MemoryRow {
22
- id: string;
23
- text: string;
24
- vector: number[];
25
- category: string;
26
- scope: string;
27
- importance: number;
28
- timestamp: number;
29
- metadata?: string;
30
- }
31
-
32
- function cosineSimilarity(a: number[], b: number[]): number {
33
- let dot = 0, normA = 0, normB = 0;
34
- for (let i = 0; i < a.length; i++) {
35
- dot += a[i] * b[i];
36
- normA += a[i] * a[i];
37
- normB += b[i] * b[i];
38
- }
39
- return dot / (Math.sqrt(normA) * Math.sqrt(normB));
40
- }
41
-
42
- function formatDate(ts: number): string {
43
- return new Date(ts).toISOString().slice(0, 10);
44
- }
45
-
46
- function percentBar(pct: number, width = 30): string {
47
- const filled = Math.round(pct / 100 * width);
48
- return "█".repeat(filled) + "░".repeat(width - filled);
49
- }
50
-
51
- async function main() {
52
- console.log("\n🏥 RecallNest 记忆体检报告");
53
- console.log("=" .repeat(60));
54
- console.log(`📅 检查时间: ${new Date().toISOString().slice(0, 19)}`);
55
- console.log(`📂 数据路径: ${DB_PATH}\n`);
56
-
57
- const db = await lancedb.connect(DB_PATH);
58
- const table = await db.openTable(TABLE_NAME);
59
-
60
- // Fetch all rows (without vectors first for speed)
61
- console.log("⏳ 正在读取全量数据...");
62
- const allRows: MemoryRow[] = await table
63
- .query()
64
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
65
- .toArray() as any;
66
-
67
- const total = allRows.length;
68
- console.log(`✅ 读取完成: ${total.toLocaleString()} 条记忆\n`);
69
-
70
- // ============================================
71
- // 1. Category 分布
72
- // ============================================
73
- console.log("📊 1. Category 分布");
74
- console.log("-".repeat(60));
75
- const catMap = new Map<string, number>();
76
- for (const row of allRows) {
77
- catMap.set(row.category, (catMap.get(row.category) || 0) + 1);
78
- }
79
- const catSorted = [...catMap.entries()].sort((a, b) => b[1] - a[1]);
80
- for (const [cat, count] of catSorted) {
81
- const pct = (count / total * 100).toFixed(1);
82
- console.log(` ${cat.padEnd(14)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${percentBar(parseFloat(pct))}`);
83
- }
84
-
85
- // ============================================
86
- // 2. Tier 分布
87
- // ============================================
88
- console.log("\n📊 2. Tier 分布");
89
- console.log("-".repeat(60));
90
- const tierMap = new Map<string, number>();
91
- let noTier = 0;
92
- for (const row of allRows) {
93
- const meta = row.metadata ? JSON.parse(row.metadata) : {};
94
- const tier = meta.tier || "unknown";
95
- if (tier === "unknown") noTier++;
96
- tierMap.set(tier, (tierMap.get(tier) || 0) + 1);
97
- }
98
- const tierSorted = [...tierMap.entries()].sort((a, b) => b[1] - a[1]);
99
- for (const [tier, count] of tierSorted) {
100
- const pct = (count / total * 100).toFixed(1);
101
- console.log(` ${tier.padEnd(14)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${percentBar(parseFloat(pct))}`);
102
- }
103
-
104
- // ============================================
105
- // 3. 死记忆(从未被 recall 命中)
106
- // ============================================
107
- console.log("\n📊 3. 死记忆分析(accessCount = 0 或无记录)");
108
- console.log("-".repeat(60));
109
- let deadCount = 0;
110
- let aliveCount = 0;
111
- let totalAccess = 0;
112
- let maxAccess = 0;
113
- let maxAccessId = "";
114
- let maxAccessText = "";
115
- const accessDistribution = new Map<string, number>();
116
-
117
- for (const row of allRows) {
118
- const meta = row.metadata ? JSON.parse(row.metadata) : {};
119
- const ac = meta.accessCount || 0;
120
- totalAccess += ac;
121
-
122
- if (ac === 0) {
123
- deadCount++;
124
- } else {
125
- aliveCount++;
126
- }
127
-
128
- if (ac > maxAccess) {
129
- maxAccess = ac;
130
- maxAccessId = row.id;
131
- maxAccessText = row.text.slice(0, 80);
132
- }
133
-
134
- // Bucket access counts
135
- let bucket: string;
136
- if (ac === 0) bucket = "0 (死记忆)";
137
- else if (ac <= 2) bucket = "1-2";
138
- else if (ac <= 5) bucket = "3-5";
139
- else if (ac <= 10) bucket = "6-10";
140
- else if (ac <= 50) bucket = "11-50";
141
- else bucket = "50+";
142
- accessDistribution.set(bucket, (accessDistribution.get(bucket) || 0) + 1);
143
- }
144
-
145
- const deadPct = (deadCount / total * 100).toFixed(1);
146
- const alivePct = (aliveCount / total * 100).toFixed(1);
147
- console.log(` 死记忆: ${deadCount.toLocaleString()} / ${total.toLocaleString()} (${deadPct}%)`);
148
- console.log(` 活记忆: ${aliveCount.toLocaleString()} / ${total.toLocaleString()} (${alivePct}%)`);
149
- console.log(` 总访问次: ${totalAccess.toLocaleString()}`);
150
- console.log(` 平均访问: ${(totalAccess / total).toFixed(2)} 次/条`);
151
- console.log(` 最热记忆: [${maxAccess}次] ${maxAccessText}...`);
152
-
153
- console.log("\n 访问次数分布:");
154
- const bucketOrder = ["0 (死记忆)", "1-2", "3-5", "6-10", "11-50", "50+"];
155
- for (const bucket of bucketOrder) {
156
- const count = accessDistribution.get(bucket) || 0;
157
- const pct = (count / total * 100).toFixed(1);
158
- console.log(` ${bucket.padEnd(14)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${percentBar(parseFloat(pct))}`);
159
- }
160
-
161
- // ============================================
162
- // 4. Importance 分布
163
- // ============================================
164
- console.log("\n📊 4. Importance 分布");
165
- console.log("-".repeat(60));
166
- const impBuckets = new Map<string, number>();
167
- let impSum = 0;
168
- for (const row of allRows) {
169
- const imp = row.importance || 0;
170
- impSum += imp;
171
- let bucket: string;
172
- if (imp < 0.2) bucket = "0.0-0.2 (低)";
173
- else if (imp < 0.4) bucket = "0.2-0.4";
174
- else if (imp < 0.6) bucket = "0.4-0.6 (中)";
175
- else if (imp < 0.8) bucket = "0.6-0.8";
176
- else bucket = "0.8-1.0 (高)";
177
- impBuckets.set(bucket, (impBuckets.get(bucket) || 0) + 1);
178
- }
179
- console.log(` 平均 importance: ${(impSum / total).toFixed(3)}`);
180
- const impOrder = ["0.0-0.2 (低)", "0.2-0.4", "0.4-0.6 (中)", "0.6-0.8", "0.8-1.0 (高)"];
181
- for (const bucket of impOrder) {
182
- const count = impBuckets.get(bucket) || 0;
183
- const pct = (count / total * 100).toFixed(1);
184
- console.log(` ${bucket.padEnd(16)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${percentBar(parseFloat(pct))}`);
185
- }
186
-
187
- // ============================================
188
- // 5. 来源分类汇总
189
- // ============================================
190
- console.log("\n📊 5. 来源分类汇总");
191
- console.log("-".repeat(60));
192
- let ccCount = 0, codexCount = 0, geminiCount = 0, memoryCount = 0, otherCount = 0;
193
- const scopeSet = new Set<string>();
194
- for (const row of allRows) {
195
- scopeSet.add(row.scope);
196
- const meta = row.metadata ? JSON.parse(row.metadata) : {};
197
- const source = meta.source || row.scope || "";
198
- if (source.startsWith("cc:") || row.scope.startsWith("cc:")) ccCount++;
199
- else if (source.startsWith("codex:") || row.scope.startsWith("codex:")) codexCount++;
200
- else if (source.startsWith("gemini:") || row.scope.startsWith("gemini:")) geminiCount++;
201
- else if (source === "memory" || row.scope === "memory" || source === "manual" || source === "agent") memoryCount++;
202
- else otherCount++;
203
- }
204
- console.log(` Claude Code: ${ccCount.toLocaleString().padStart(6)} (${(ccCount/total*100).toFixed(1)}%)`);
205
- console.log(` Codex: ${codexCount.toLocaleString().padStart(6)} (${(codexCount/total*100).toFixed(1)}%)`);
206
- console.log(` Gemini: ${geminiCount.toLocaleString().padStart(6)} (${(geminiCount/total*100).toFixed(1)}%)`);
207
- console.log(` 手动记忆: ${memoryCount.toLocaleString().padStart(6)} (${(memoryCount/total*100).toFixed(1)}%)`);
208
- console.log(` 其他: ${otherCount.toLocaleString().padStart(6)} (${(otherCount/total*100).toFixed(1)}%)`);
209
- console.log(` 独立 scope 数: ${scopeSet.size}`);
210
-
211
- // ============================================
212
- // 6. 年龄分布
213
- // ============================================
214
- console.log("\n📊 6. 年龄分布");
215
- console.log("-".repeat(60));
216
- const now = Date.now();
217
- const ageBuckets = new Map<string, number>();
218
- let oldest = Infinity, newest = 0;
219
- for (const row of allRows) {
220
- const ts = row.timestamp;
221
- if (ts < oldest) oldest = ts;
222
- if (ts > newest) newest = ts;
223
- const ageHours = (now - ts) / (1000 * 60 * 60);
224
- let bucket: string;
225
- if (ageHours < 24) bucket = "< 1 天";
226
- else if (ageHours < 24 * 7) bucket = "1-7 天";
227
- else if (ageHours < 24 * 30) bucket = "7-30 天";
228
- else if (ageHours < 24 * 90) bucket = "30-90 天";
229
- else bucket = "> 90 天";
230
- ageBuckets.set(bucket, (ageBuckets.get(bucket) || 0) + 1);
231
- }
232
- console.log(` 最早: ${formatDate(oldest)}`);
233
- console.log(` 最新: ${formatDate(newest)}`);
234
- const ageOrder = ["< 1 天", "1-7 天", "7-30 天", "30-90 天", "> 90 天"];
235
- for (const bucket of ageOrder) {
236
- const count = ageBuckets.get(bucket) || 0;
237
- const pct = (count / total * 100).toFixed(1);
238
- console.log(` ${bucket.padEnd(12)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${percentBar(parseFloat(pct))}`);
239
- }
240
-
241
- // ============================================
242
- // 7. 近似重复检测(采样)
243
- // ============================================
244
- console.log("\n📊 7. 近似重复检测(采样 500 条,cosine > 0.95)");
245
- console.log("-".repeat(60));
246
- console.log(" ⏳ 正在读取向量数据(采样)...");
247
-
248
- // Sample 500 random rows with vectors
249
- const sampleSize = 500;
250
- const sampleRows: MemoryRow[] = await table
251
- .query()
252
- .select(["id", "text", "vector", "category"])
253
- .limit(sampleSize)
254
- .toArray() as any;
255
-
256
- let dupPairs = 0;
257
- const dupExamples: Array<{sim: number, textA: string, textB: string}> = [];
258
-
259
- for (let i = 0; i < sampleRows.length; i++) {
260
- for (let j = i + 1; j < sampleRows.length; j++) {
261
- if (sampleRows[i].vector && sampleRows[j].vector) {
262
- const sim = cosineSimilarity(sampleRows[i].vector, sampleRows[j].vector);
263
- if (sim > 0.95) {
264
- dupPairs++;
265
- if (dupExamples.length < 3) {
266
- dupExamples.push({
267
- sim,
268
- textA: sampleRows[i].text.slice(0, 60),
269
- textB: sampleRows[j].text.slice(0, 60),
270
- });
271
- }
272
- }
273
- }
274
- }
275
- }
276
-
277
- const totalPairs = sampleSize * (sampleSize - 1) / 2;
278
- const dupRate = (dupPairs / totalPairs * 100).toFixed(3);
279
- console.log(` 采样对数: ${totalPairs.toLocaleString()}`);
280
- console.log(` 高相似对: ${dupPairs} (>${dupRate}%)`);
281
- console.log(` 推算全库: ~${Math.round(dupPairs / totalPairs * total * (total-1) / 2).toLocaleString()} 对潜在重复`);
282
-
283
- if (dupExamples.length > 0) {
284
- console.log("\n 示例重复对:");
285
- for (const ex of dupExamples) {
286
- console.log(` [sim=${ex.sim.toFixed(3)}]`);
287
- console.log(` A: ${ex.textA}...`);
288
- console.log(` B: ${ex.textB}...`);
289
- }
290
- }
291
-
292
- // ============================================
293
- // 8. 综合诊断
294
- // ============================================
295
- console.log("\n" + "=".repeat(60));
296
- console.log("🩺 综合诊断");
297
- console.log("=".repeat(60));
298
-
299
- const issues: string[] = [];
300
- const healthy: string[] = [];
301
-
302
- // Check dead memory ratio
303
- const deadRatio = deadCount / total;
304
- if (deadRatio > 0.9) {
305
- issues.push(`🔴 ${deadPct}% 记忆从未被访问(死记忆率过高)`);
306
- } else if (deadRatio > 0.7) {
307
- issues.push(`🟡 ${deadPct}% 记忆从未被访问(死记忆率偏高)`);
308
- } else {
309
- healthy.push(`✅ 死记忆率 ${deadPct}% 在合理范围`);
310
- }
311
-
312
- // Check category balance
313
- const factRatio = (catMap.get("fact") || 0) / total;
314
- if (factRatio > 0.8) {
315
- issues.push(`🟡 fact 类别占 ${(factRatio*100).toFixed(1)}%,结构化记忆(entities/patterns/cases)占比偏低`);
316
- } else {
317
- healthy.push(`✅ 类别分布均衡`);
318
- }
319
-
320
- // Check tier distribution
321
- const coreCount = tierMap.get("core") || 0;
322
- const workingCount = tierMap.get("working") || 0;
323
- if (coreCount < 10) {
324
- issues.push(`🟡 core tier 仅 ${coreCount} 条,高价值记忆太少`);
325
- } else {
326
- healthy.push(`✅ core tier 有 ${coreCount} 条高价值记忆`);
327
- }
328
-
329
- // Check dup rate
330
- if (parseFloat(dupRate) > 1) {
331
- issues.push(`🟡 采样重复率 ${dupRate}%,全库可能存在大量近似重复`);
332
- } else {
333
- healthy.push(`✅ 采样重复率 ${dupRate}%,重复可控`);
334
- }
335
-
336
- for (const h of healthy) console.log(` ${h}`);
337
- for (const i of issues) console.log(` ${i}`);
338
-
339
- console.log("\n" + "=".repeat(60));
340
- console.log("📋 体检完成\n");
341
- }
342
-
343
- main().catch(console.error);