recallnest 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. package/.claude-plugin/CLAUDE.md +29 -0
  2. package/.claude-plugin/marketplace.json +18 -0
  3. package/.env.example +3 -0
  4. package/.github/workflows/ci.yml +28 -0
  5. package/.mcp.json +9 -0
  6. package/CHANGELOG.md +51 -0
  7. package/CLAUDE.md +42 -0
  8. package/DEMO.md +94 -0
  9. package/EVAL.md +89 -0
  10. package/FAILURES.md +75 -0
  11. package/LICENSE +21 -0
  12. package/OPERATIONS.md +59 -0
  13. package/README.md +427 -0
  14. package/README_CN.md +425 -0
  15. package/ROADMAP.md +198 -0
  16. package/assets/recallnest-workbench.png +0 -0
  17. package/assets/ui/app.js +807 -0
  18. package/assets/ui/index.html +154 -0
  19. package/assets/ui/styles.css +605 -0
  20. package/assets/wechat-qr.jpg +0 -0
  21. package/config.json.example +38 -0
  22. package/daily-recall.sh +87 -0
  23. package/daily-reflections/2026-03-05.md +15 -0
  24. package/daily-reflections/2026-03-06.md +15 -0
  25. package/daily-reflections/2026-03-07.md +7 -0
  26. package/daily-reflections/2026-03-08.md +34 -0
  27. package/daily-reflections/2026-03-09.md +35 -0
  28. package/daily-reflections/2026-03-10.md +36 -0
  29. package/daily-reflections/2026-03-11.md +35 -0
  30. package/daily-reflections/2026-03-12.md +36 -0
  31. package/daily-reflections/2026-03-13.md +35 -0
  32. package/daily-reflections/2026-03-14.md +35 -0
  33. package/daily-reflections/2026-03-15.md +35 -0
  34. package/daily-reflections/2026-03-16.md +41 -0
  35. package/daily-reflections/2026-03-17.md +10 -0
  36. package/daily-reflections/2026-03-18.md +10 -0
  37. package/daily-reflections/2026-03-19.md +10 -0
  38. package/daily-reflections/2026-03-20.md +10 -0
  39. package/daily-reflections/2026-03-21.md +10 -0
  40. package/daily-reflections/2026-03-22.md +10 -0
  41. package/daily-reflections/2026-03-23.md +10 -0
  42. package/daily-reflections/2026-03-24.md +10 -0
  43. package/daily-reflections/2026-03-25.md +32 -0
  44. package/daily-reflections/2026-03-26.md +37 -0
  45. package/daily-reflections/2026-03-27.md +35 -0
  46. package/daily-reflections/2026-03-28.md +35 -0
  47. package/daily-reflections/2026-03-29.md +35 -0
  48. package/daily-reflections/2026-03-30.md +36 -0
  49. package/daily-reflections/2026-03-31.md +35 -0
  50. package/daily-reflections/2026-04-01.md +35 -0
  51. package/docs/api-reference.md +1291 -0
  52. package/docs/architecture.md +168 -0
  53. package/docs/memory-boundary-contract.md +226 -0
  54. package/docs/memory-categories.md +119 -0
  55. package/docs/pipeline-audit-ultramemory.md +177 -0
  56. package/docs/plans/2026-03-08-query-expansion.md +270 -0
  57. package/docs/plans/2026-03-16-continuity-layer.md +196 -0
  58. package/docs/plans/mcp-tiering-plan.md +126 -0
  59. package/docs/research/recall-quality-audit-2026-03-22.md +153 -0
  60. package/docs/self-evolution.md +198 -0
  61. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +782 -0
  62. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +77 -0
  63. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +95 -0
  64. package/eval/cases.json +85 -0
  65. package/eval/continuity/case-seeds.json +90 -0
  66. package/eval/continuity/cases.json +302 -0
  67. package/eval/continuity/memory-seeds.json +26 -0
  68. package/eval/continuity/pattern-seeds.json +100 -0
  69. package/eval/continuity/results-2026-03-16.md +117 -0
  70. package/eval/continuity/results-2026-03-19.md +453 -0
  71. package/eval/continuity/runbook.md +110 -0
  72. package/eval/continuity/seed-cases.md +54 -0
  73. package/eval/reports/2026-03-06-abstract-query-baseline.md +96 -0
  74. package/eval/reports/2026-03-06-aws-ssh-baseline.md +82 -0
  75. package/eval/reports/2026-03-06-baseline.md +82 -0
  76. package/eval/reports/query-expansion.md +138 -0
  77. package/integrations/README.md +57 -0
  78. package/integrations/claude-code/README.md +133 -0
  79. package/integrations/claude-code/claude-md-snippet.md +31 -0
  80. package/integrations/claude-code/setup.sh +66 -0
  81. package/integrations/claude-code/smoke-continuity.sh +452 -0
  82. package/integrations/codex/README.md +49 -0
  83. package/integrations/codex/agents-md-snippet.md +31 -0
  84. package/integrations/codex/setup.sh +58 -0
  85. package/integrations/examples/claude-agent-sdk/memory-agent.ts +182 -0
  86. package/integrations/examples/langchain/memory-chain.py +111 -0
  87. package/integrations/examples/openai-agents-sdk/memory-agent.py +109 -0
  88. package/integrations/gemini-cli/README.md +57 -0
  89. package/integrations/gemini-cli/gemini-md-snippet.md +31 -0
  90. package/integrations/gemini-cli/setup.sh +64 -0
  91. package/integrations/shared/install-managed-block.sh +51 -0
  92. package/lm +19 -0
  93. package/package.json +50 -0
  94. package/plan-v1.2-distributable.md +105 -0
  95. package/scripts/archive-distilled-junk.ts +116 -0
  96. package/scripts/backfill-l0l1l2.ts +98 -0
  97. package/scripts/blind-scan.ts +588 -0
  98. package/scripts/cleanup-step1-exact-dups.ts +117 -0
  99. package/scripts/cleanup-step2-template-dups.ts +113 -0
  100. package/scripts/cleanup-step3-stale-content.ts +149 -0
  101. package/scripts/cleanup-step4-orphan-scopes.ts +131 -0
  102. package/scripts/delete-distilled-junk.ts +88 -0
  103. package/scripts/distill-facts.ts +305 -0
  104. package/scripts/health-check.ts +343 -0
  105. package/scripts/incremental-ingest.sh +71 -0
  106. package/scripts/mlp-health-check.ts +382 -0
  107. package/scripts/pull-claude-desktop.ts +515 -0
  108. package/scripts/re-ingest-cc.sh +66 -0
  109. package/scripts/rebalance-memory-health.ts +181 -0
  110. package/scripts/run-desktop-ingest.sh +11 -0
  111. package/scripts/start-server.sh +12 -0
  112. package/scripts/viz-dashboard.html +423 -0
  113. package/scripts/viz-export.ts +703 -0
  114. package/src/__tests__/access-tracker-novelty.test.ts +228 -0
  115. package/src/__tests__/auto-consolidation.test.ts +168 -0
  116. package/src/__tests__/auto-recall.test.ts +131 -0
  117. package/src/__tests__/batch-dedup.test.ts +65 -0
  118. package/src/__tests__/capture-engine.test.ts +1142 -0
  119. package/src/__tests__/capture-heuristic.test.ts +171 -0
  120. package/src/__tests__/cascade-forget.test.ts +210 -0
  121. package/src/__tests__/category-min-scores.test.ts +33 -0
  122. package/src/__tests__/confidence-tracker.test.ts +196 -0
  123. package/src/__tests__/conflict-advisor.test.ts +206 -0
  124. package/src/__tests__/conflict-escalation.test.ts +149 -0
  125. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  126. package/src/__tests__/conflict-output.test.ts +85 -0
  127. package/src/__tests__/conflict-store.test.ts +374 -0
  128. package/src/__tests__/consolidation-engine.test.ts +181 -0
  129. package/src/__tests__/context-composer-pins.test.ts +64 -0
  130. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  131. package/src/__tests__/context-renderer.test.ts +91 -0
  132. package/src/__tests__/continuity-eval.test.ts +275 -0
  133. package/src/__tests__/continuity-schema.test.ts +97 -0
  134. package/src/__tests__/core-recall-eval.test.ts +235 -0
  135. package/src/__tests__/core-summary.test.ts +109 -0
  136. package/src/__tests__/dedup-actions.test.ts +58 -0
  137. package/src/__tests__/deterministic-id.test.ts +39 -0
  138. package/src/__tests__/doctor.test.ts +303 -0
  139. package/src/__tests__/drill-down.test.ts +108 -0
  140. package/src/__tests__/embedder.test.ts +122 -0
  141. package/src/__tests__/entity-resolver.test.ts +58 -0
  142. package/src/__tests__/event-segmenter.test.ts +153 -0
  143. package/src/__tests__/hotness-blend.test.ts +76 -0
  144. package/src/__tests__/ingest.test.ts +360 -0
  145. package/src/__tests__/kg-extractor.test.ts +184 -0
  146. package/src/__tests__/kg-store.test.ts +198 -0
  147. package/src/__tests__/llm-consolidation.test.ts +226 -0
  148. package/src/__tests__/memory-boundaries.test.ts +237 -0
  149. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  150. package/src/__tests__/memory-output.test.ts +215 -0
  151. package/src/__tests__/multi-vector.test.ts +242 -0
  152. package/src/__tests__/ppr-traversal.test.ts +256 -0
  153. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  154. package/src/__tests__/preference-matcher.test.ts +230 -0
  155. package/src/__tests__/preference-slots.test.ts +106 -0
  156. package/src/__tests__/prospective-memory.test.ts +283 -0
  157. package/src/__tests__/query-entity-detector.test.ts +171 -0
  158. package/src/__tests__/query-expander.test.ts +34 -0
  159. package/src/__tests__/recall-mode.test.ts +64 -0
  160. package/src/__tests__/result-synthesizer.test.ts +198 -0
  161. package/src/__tests__/resume-context.test.ts +4743 -0
  162. package/src/__tests__/retrieval-trace.test.ts +213 -0
  163. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  164. package/src/__tests__/retroactive-boost.test.ts +223 -0
  165. package/src/__tests__/scope-inventory.test.ts +200 -0
  166. package/src/__tests__/scope-policy.test.ts +69 -0
  167. package/src/__tests__/session-checkpoint.test.ts +259 -0
  168. package/src/__tests__/session-gc.test.ts +215 -0
  169. package/src/__tests__/store-scope-audit.test.ts +81 -0
  170. package/src/__tests__/temporal-parser.test.ts +241 -0
  171. package/src/__tests__/term-registry.test.ts +121 -0
  172. package/src/__tests__/tool-output-compressor.test.ts +133 -0
  173. package/src/__tests__/version-manager.test.ts +213 -0
  174. package/src/__tests__/workflow-observation.test.ts +400 -0
  175. package/src/access-tracker.ts +316 -0
  176. package/src/adaptive-retrieval.ts +64 -0
  177. package/src/api-server.ts +667 -0
  178. package/src/asset-sync.ts +79 -0
  179. package/src/auto-consolidation.ts +133 -0
  180. package/src/auto-gc.ts +124 -0
  181. package/src/auto-recall.ts +95 -0
  182. package/src/capture-engine.ts +931 -0
  183. package/src/capture-heuristic.ts +177 -0
  184. package/src/cascade-forget.ts +135 -0
  185. package/src/chunker.ts +253 -0
  186. package/src/cli.ts +1917 -0
  187. package/src/confidence-tracker.ts +166 -0
  188. package/src/conflict-advisor.ts +433 -0
  189. package/src/conflict-engine.ts +181 -0
  190. package/src/conflict-escalation.ts +138 -0
  191. package/src/conflict-lifecycle.ts +67 -0
  192. package/src/conflict-output.ts +218 -0
  193. package/src/conflict-schema.ts +130 -0
  194. package/src/conflict-store.ts +127 -0
  195. package/src/consolidation-engine.ts +256 -0
  196. package/src/context-composer-pins.ts +183 -0
  197. package/src/context-composer-queries.ts +121 -0
  198. package/src/context-composer-scope.ts +56 -0
  199. package/src/context-composer-stable-selection.ts +200 -0
  200. package/src/context-composer-stable.ts +217 -0
  201. package/src/context-composer-task-fallbacks.ts +36 -0
  202. package/src/context-composer-task-ranking.ts +197 -0
  203. package/src/context-composer-task-results.ts +137 -0
  204. package/src/context-composer-task-selection.ts +342 -0
  205. package/src/context-composer-text.ts +57 -0
  206. package/src/context-composer.ts +300 -0
  207. package/src/context-renderer.ts +179 -0
  208. package/src/decay-engine.ts +189 -0
  209. package/src/doctor.ts +543 -0
  210. package/src/embedder.ts +557 -0
  211. package/src/entity-resolver.ts +131 -0
  212. package/src/eval.ts +632 -0
  213. package/src/event-segmenter.ts +216 -0
  214. package/src/ingest.ts +1822 -0
  215. package/src/kg-extractor.ts +217 -0
  216. package/src/kg-store.ts +353 -0
  217. package/src/legacy-scope-review.ts +53 -0
  218. package/src/llm-client.ts +629 -0
  219. package/src/llm-consolidation.ts +168 -0
  220. package/src/mcp-server.ts +1337 -0
  221. package/src/memory-assets.ts +475 -0
  222. package/src/memory-boundaries.ts +354 -0
  223. package/src/memory-health-rebalance.ts +220 -0
  224. package/src/memory-output.ts +474 -0
  225. package/src/memory-schema.ts +184 -0
  226. package/src/multi-vector.ts +234 -0
  227. package/src/noise-filter.ts +155 -0
  228. package/src/ppr-traversal.ts +279 -0
  229. package/src/preference-matcher.ts +174 -0
  230. package/src/preference-slots.ts +259 -0
  231. package/src/prospective-memory.ts +198 -0
  232. package/src/query-entity-detector.ts +207 -0
  233. package/src/query-expander.ts +98 -0
  234. package/src/result-synthesizer.ts +114 -0
  235. package/src/retrieval-profiles.ts +101 -0
  236. package/src/retrieval-trace.ts +94 -0
  237. package/src/retriever.ts +1338 -0
  238. package/src/retroactive-boost.ts +154 -0
  239. package/src/rif.ts +73 -0
  240. package/src/runtime-config.ts +172 -0
  241. package/src/schema-utils.ts +56 -0
  242. package/src/scope-inventory.ts +310 -0
  243. package/src/scope-policy.ts +142 -0
  244. package/src/session-engine.ts +168 -0
  245. package/src/session-output.ts +96 -0
  246. package/src/session-schema.ts +60 -0
  247. package/src/session-store.ts +293 -0
  248. package/src/stderr-log.ts +16 -0
  249. package/src/store.ts +903 -0
  250. package/src/temporal-parser.ts +318 -0
  251. package/src/term-registry.ts +791 -0
  252. package/src/tool-output-compressor.ts +115 -0
  253. package/src/tracker.ts +57 -0
  254. package/src/ui-server.ts +430 -0
  255. package/src/version-manager.ts +172 -0
  256. package/src/workflow-observation-engine.ts +316 -0
  257. package/src/workflow-observation-managed.ts +83 -0
  258. package/src/workflow-observation-output.ts +86 -0
  259. package/src/workflow-observation-schema.ts +87 -0
  260. package/src/workflow-observation-scope-review.ts +409 -0
  261. package/src/workflow-observation-store.ts +69 -0
@@ -0,0 +1,588 @@
1
+ /**
2
+ * 盲扫脚本 — 模块 4:开放式探索
3
+ * 不带预设地分析 38K 条记忆的文本内容
4
+ * 只读,不写入任何内容
5
+ *
6
+ * 用法:bun run scripts/blind-scan.ts [lancedb-dir]
7
+ */
8
+
9
+ import lancedb from "@lancedb/lancedb";
10
+
11
+ const DB_PATH = process.argv[2] || "./data/lancedb";
12
+ const TABLE_NAME = "memories";
13
+
14
+ interface MemoryRow {
15
+ id: string;
16
+ text: string;
17
+ vector: number[];
18
+ category: string;
19
+ scope: string;
20
+ importance: number;
21
+ timestamp: number;
22
+ metadata?: string;
23
+ }
24
+
25
+ function cosineSimilarity(a: number[], b: number[]): number {
26
+ let dot = 0, normA = 0, normB = 0;
27
+ for (let i = 0; i < a.length; i++) {
28
+ dot += a[i] * b[i];
29
+ normA += a[i] * a[i];
30
+ normB += b[i] * b[i];
31
+ }
32
+ return dot / (Math.sqrt(normA) * Math.sqrt(normB));
33
+ }
34
+
35
+ function formatDate(ts: number): string {
36
+ return new Date(ts).toISOString().slice(0, 10);
37
+ }
38
+
39
+ // Simple CJK-aware tokenizer
40
+ function tokenize(text: string): string[] {
41
+ const tokens: string[] = [];
42
+ // Remove URLs
43
+ const cleaned = text.replace(/https?:\/\/\S+/g, " ");
44
+ // Split into words (English) and characters (CJK)
45
+ const parts = cleaned.match(/[\u4e00-\u9fff\u3400-\u4dbf]+|[a-zA-Z][a-zA-Z0-9_'-]+/g) || [];
46
+ for (const part of parts) {
47
+ if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(part)) {
48
+ // CJK: bigrams
49
+ for (let i = 0; i < part.length - 1; i++) {
50
+ tokens.push(part.slice(i, i + 2));
51
+ }
52
+ } else if (part.length > 1) {
53
+ tokens.push(part.toLowerCase());
54
+ }
55
+ }
56
+ return tokens;
57
+ }
58
+
59
+ // Stopwords (minimal)
60
+ const STOPWORDS = new Set([
61
+ "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
62
+ "have", "has", "had", "do", "does", "did", "will", "would", "could",
63
+ "should", "may", "might", "shall", "can", "need", "must",
64
+ "to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
65
+ "into", "through", "during", "before", "after", "above", "below",
66
+ "between", "out", "off", "over", "under", "again", "further",
67
+ "then", "once", "here", "there", "when", "where", "why", "how",
68
+ "all", "each", "every", "both", "few", "more", "most", "other",
69
+ "some", "such", "no", "nor", "not", "only", "own", "same", "so",
70
+ "than", "too", "very", "just", "because", "but", "and", "or", "if",
71
+ "while", "about", "up", "its", "it", "this", "that", "these", "those",
72
+ "he", "she", "they", "we", "you", "me", "him", "her", "us", "them",
73
+ "my", "your", "his", "our", "their", "what", "which", "who", "whom",
74
+ "also", "been", "like", "get", "got", "make", "made", "use", "used",
75
+ ]);
76
+
77
+ async function main() {
78
+ console.log("\n🔍 盲扫报告 — RecallNest 开放式探索");
79
+ console.log("=".repeat(70));
80
+ console.log(`📅 时间: ${new Date().toISOString().slice(0, 19)}`);
81
+ console.log(`📂 路径: ${DB_PATH}\n`);
82
+
83
+ const db = await lancedb.connect(DB_PATH);
84
+ const table = await db.openTable(TABLE_NAME);
85
+
86
+ console.log("⏳ 读取全量数据...");
87
+ const allRows: MemoryRow[] = await table
88
+ .query()
89
+ .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
90
+ .toArray() as any;
91
+
92
+ const total = allRows.length;
93
+ console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
94
+
95
+ const parsedRows = allRows.map(row => ({
96
+ ...row,
97
+ meta: row.metadata ? JSON.parse(row.metadata) : {},
98
+ }));
99
+
100
+ // ============================================
101
+ // 1. 文本长度分布
102
+ // ============================================
103
+ console.log("📊 1. 文本长度分布");
104
+ console.log("-".repeat(70));
105
+ const lengths = parsedRows.map(r => r.text.length);
106
+ lengths.sort((a, b) => a - b);
107
+ const avgLen = lengths.reduce((s, l) => s + l, 0) / total;
108
+ const medianLen = lengths[Math.floor(total / 2)];
109
+ const p95Len = lengths[Math.floor(total * 0.95)];
110
+ const p99Len = lengths[Math.floor(total * 0.99)];
111
+
112
+ console.log(` 最短: ${lengths[0]} 字符`);
113
+ console.log(` 中位: ${medianLen} 字符`);
114
+ console.log(` 平均: ${Math.round(avgLen)} 字符`);
115
+ console.log(` P95: ${p95Len} 字符`);
116
+ console.log(` P99: ${p99Len} 字符`);
117
+ console.log(` 最长: ${lengths[total - 1]} 字符`);
118
+
119
+ // Length buckets
120
+ const lenBuckets = new Map<string, number>();
121
+ const lenRanges: Array<[number, number, string]> = [
122
+ [0, 50, "0-50"],
123
+ [50, 100, "50-100"],
124
+ [100, 200, "100-200"],
125
+ [200, 500, "200-500"],
126
+ [500, 1000, "500-1K"],
127
+ [1000, 2000, "1K-2K"],
128
+ [2000, 5000, "2K-5K"],
129
+ [5000, Infinity, "5K+"],
130
+ ];
131
+ for (const len of lengths) {
132
+ for (const [lo, hi, label] of lenRanges) {
133
+ if (len >= lo && len < hi) {
134
+ lenBuckets.set(label, (lenBuckets.get(label) || 0) + 1);
135
+ break;
136
+ }
137
+ }
138
+ }
139
+ console.log("\n 分布:");
140
+ for (const [, , label] of lenRanges) {
141
+ const count = lenBuckets.get(label) || 0;
142
+ const pct = (count / total * 100).toFixed(1);
143
+ const bar = "█".repeat(Math.round(parseFloat(pct) / 100 * 40));
144
+ console.log(` ${label.padEnd(10)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${bar}`);
145
+ }
146
+
147
+ // Extremely short texts (likely garbage)
148
+ const ultraShort = parsedRows.filter(r => r.text.length < 20);
149
+ if (ultraShort.length > 0) {
150
+ console.log(`\n ⚠️ 超短文本 (<20 字符): ${ultraShort.length} 条`);
151
+ console.log(" 示例:");
152
+ for (const r of ultraShort.slice(0, 5)) {
153
+ console.log(` [${r.category}] "${r.text}"`);
154
+ }
155
+ }
156
+
157
+ // ============================================
158
+ // 2. 语言混合分析
159
+ // ============================================
160
+ console.log("\n📊 2. 语言混合分析");
161
+ console.log("-".repeat(70));
162
+ let pureEn = 0, pureCn = 0, mixed = 0, other = 0;
163
+ for (const row of parsedRows) {
164
+ const hasCJK = /[\u4e00-\u9fff]/.test(row.text);
165
+ const hasEn = /[a-zA-Z]{3,}/.test(row.text);
166
+ if (hasCJK && hasEn) mixed++;
167
+ else if (hasCJK) pureCn++;
168
+ else if (hasEn) pureEn++;
169
+ else other++;
170
+ }
171
+ console.log(` 纯英文: ${pureEn.toLocaleString()} (${(pureEn/total*100).toFixed(1)}%)`);
172
+ console.log(` 纯中文: ${pureCn.toLocaleString()} (${(pureCn/total*100).toFixed(1)}%)`);
173
+ console.log(` 中英混合: ${mixed.toLocaleString()} (${(mixed/total*100).toFixed(1)}%)`);
174
+ console.log(` 其他: ${other.toLocaleString()} (${(other/total*100).toFixed(1)}%)`);
175
+
176
+ // ============================================
177
+ // 3. 高频词分析(分语言)
178
+ // ============================================
179
+ console.log("\n📊 3. 高频词分析");
180
+ console.log("-".repeat(70));
181
+ const wordFreq = new Map<string, number>();
182
+ for (const row of parsedRows) {
183
+ const tokens = tokenize(row.text);
184
+ for (const t of tokens) {
185
+ if (!STOPWORDS.has(t) && t.length > 1) {
186
+ wordFreq.set(t, (wordFreq.get(t) || 0) + 1);
187
+ }
188
+ }
189
+ }
190
+ const topWords = [...wordFreq.entries()]
191
+ .sort((a, b) => b[1] - a[1])
192
+ .slice(0, 40);
193
+
194
+ console.log(" Top 40 词频(去停用词):");
195
+ for (let i = 0; i < topWords.length; i++) {
196
+ const [word, count] = topWords[i];
197
+ const pctOfDocs = (count / total * 100).toFixed(1);
198
+ console.log(` ${(i+1).toString().padStart(2)}. ${word.padEnd(20)} ${count.toString().padStart(6)} (${pctOfDocs}% 文档)`);
199
+ }
200
+
201
+ // ============================================
202
+ // 4. 文本前缀聚类(发现模板化内容)
203
+ // ============================================
204
+ console.log("\n📊 4. 文本前缀聚类(发现模板化内容)");
205
+ console.log("-".repeat(70));
206
+ const prefixMap = new Map<string, number>();
207
+ for (const row of parsedRows) {
208
+ // Take first 40 chars as prefix fingerprint
209
+ const prefix = row.text.slice(0, 40).trim();
210
+ prefixMap.set(prefix, (prefixMap.get(prefix) || 0) + 1);
211
+ }
212
+ const repeatedPrefixes = [...prefixMap.entries()]
213
+ .filter(([, count]) => count >= 3)
214
+ .sort((a, b) => b[1] - a[1]);
215
+
216
+ console.log(` 独立前缀数: ${prefixMap.size.toLocaleString()}`);
217
+ console.log(` 重复 ≥3 次的前缀: ${repeatedPrefixes.length}`);
218
+
219
+ if (repeatedPrefixes.length > 0) {
220
+ console.log("\n Top 20 重复前缀:");
221
+ for (const [prefix, count] of repeatedPrefixes.slice(0, 20)) {
222
+ console.log(` [×${count.toString().padStart(4)}] ${prefix}...`);
223
+ }
224
+ }
225
+
226
+ // ============================================
227
+ // 5. Scope × Category 交叉分析
228
+ // ============================================
229
+ console.log("\n📊 5. Scope × Category 交叉分析(Top 10 Scope)");
230
+ console.log("-".repeat(70));
231
+ const scopeCat = new Map<string, Map<string, number>>();
232
+ for (const row of parsedRows) {
233
+ const s = row.scope || "(empty)";
234
+ if (!scopeCat.has(s)) scopeCat.set(s, new Map());
235
+ const catMap = scopeCat.get(s)!;
236
+ catMap.set(row.category, (catMap.get(row.category) || 0) + 1);
237
+ }
238
+ const topScopes = [...scopeCat.entries()]
239
+ .map(([scope, cats]) => {
240
+ const scopeTotal = [...cats.values()].reduce((s, c) => s + c, 0);
241
+ return { scope, total: scopeTotal, cats };
242
+ })
243
+ .sort((a, b) => b.total - a.total)
244
+ .slice(0, 10);
245
+
246
+ const allCats = [...new Set(parsedRows.map(r => r.category))].sort();
247
+ // Header
248
+ let header = " Scope".padEnd(35);
249
+ for (const cat of allCats) header += cat.slice(0, 6).padStart(8);
250
+ header += " Total".padStart(8);
251
+ console.log(header);
252
+
253
+ for (const { scope, total: scopeTotal, cats } of topScopes) {
254
+ const label = scope.length > 30 ? scope.slice(0, 27) + "..." : scope;
255
+ let line = ` ${label.padEnd(33)}`;
256
+ for (const cat of allCats) {
257
+ line += (cats.get(cat) || 0).toString().padStart(8);
258
+ }
259
+ line += scopeTotal.toString().padStart(8);
260
+ console.log(line);
261
+ }
262
+
263
+ // ============================================
264
+ // 6. 时间模式分析
265
+ // ============================================
266
+ console.log("\n📊 6. 时间模式分析");
267
+ console.log("-".repeat(70));
268
+
269
+ // By month
270
+ const monthMap = new Map<string, number>();
271
+ // By hour of day
272
+ const hourMap = new Map<number, number>();
273
+ // By day of week
274
+ const dowMap = new Map<number, number>();
275
+ const DOW_NAMES = ["周日", "周一", "周二", "周三", "周四", "周五", "周六"];
276
+
277
+ for (const row of parsedRows) {
278
+ const d = new Date(row.timestamp);
279
+ const month = d.toISOString().slice(0, 7);
280
+ monthMap.set(month, (monthMap.get(month) || 0) + 1);
281
+ hourMap.set(d.getHours(), (hourMap.get(d.getHours()) || 0) + 1);
282
+ dowMap.set(d.getDay(), (dowMap.get(d.getDay()) || 0) + 1);
283
+ }
284
+
285
+ // Monthly trend
286
+ const months = [...monthMap.entries()].sort();
287
+ console.log(" 月度趋势:");
288
+ const maxMonthCount = Math.max(...months.map(m => m[1]));
289
+ for (const [month, count] of months) {
290
+ const bar = "█".repeat(Math.round(count / maxMonthCount * 30));
291
+ console.log(` ${month} ${count.toString().padStart(6)} ${bar}`);
292
+ }
293
+
294
+ // Hourly pattern
295
+ console.log("\n 小时分布:");
296
+ const maxHourCount = Math.max(...[...hourMap.values()]);
297
+ for (let h = 0; h < 24; h++) {
298
+ const count = hourMap.get(h) || 0;
299
+ const bar = "█".repeat(Math.round(count / maxHourCount * 20));
300
+ console.log(` ${h.toString().padStart(2)}:00 ${count.toString().padStart(5)} ${bar}`);
301
+ }
302
+
303
+ // Day of week
304
+ console.log("\n 星期分布:");
305
+ for (let d = 0; d < 7; d++) {
306
+ const count = dowMap.get(d) || 0;
307
+ const pct = (count / total * 100).toFixed(1);
308
+ console.log(` ${DOW_NAMES[d]} ${count.toString().padStart(6)} (${pct}%)`);
309
+ }
310
+
311
+ // ============================================
312
+ // 7. 过时内容检测
313
+ // ============================================
314
+ console.log("\n📊 7. 过时内容检测");
315
+ console.log("-".repeat(70));
316
+
317
+ // Look for date references in text
318
+ const yearRefs = new Map<string, number>();
319
+ for (const row of parsedRows) {
320
+ const yearPattern = /20(2[0-6])/g;
321
+ let match;
322
+ while ((match = yearPattern.exec(row.text)) !== null) {
323
+ const year = `20${match[1]}`;
324
+ yearRefs.set(year, (yearRefs.get(year) || 0) + 1);
325
+ }
326
+ }
327
+ console.log(" 文本中提到的年份:");
328
+ for (const [year, count] of [...yearRefs.entries()].sort()) {
329
+ console.log(` ${year}: ${count.toLocaleString()} 次`);
330
+ }
331
+
332
+ // Look for potentially outdated keywords
333
+ const outdatedKeywords = [
334
+ "deprecated", "removed", "old", "legacy", "TODO", "FIXME", "HACK",
335
+ "temporary", "workaround", "临时", "废弃", "已删除", "待修复",
336
+ ];
337
+ console.log("\n 过时关键词扫描:");
338
+ for (const kw of outdatedKeywords) {
339
+ const count = parsedRows.filter(r =>
340
+ r.text.toLowerCase().includes(kw.toLowerCase())
341
+ ).length;
342
+ if (count > 0) {
343
+ console.log(` "${kw}": ${count} 条`);
344
+ }
345
+ }
346
+
347
+ // ============================================
348
+ // 8. 文本精确重复检测
349
+ // ============================================
350
+ console.log("\n📊 8. 精确/近似文本重复");
351
+ console.log("-".repeat(70));
352
+
353
+ // Exact text duplicates
354
+ const textMap = new Map<string, number>();
355
+ for (const row of parsedRows) {
356
+ const normalized = row.text.trim().toLowerCase();
357
+ textMap.set(normalized, (textMap.get(normalized) || 0) + 1);
358
+ }
359
+ const exactDups = [...textMap.entries()].filter(([, count]) => count > 1);
360
+ const totalDupRows = exactDups.reduce((s, [, c]) => s + c, 0);
361
+ console.log(` 精确重复组: ${exactDups.length}`);
362
+ console.log(` 重复行总数: ${totalDupRows} / ${total} (${(totalDupRows/total*100).toFixed(1)}%)`);
363
+
364
+ if (exactDups.length > 0) {
365
+ const topDups = exactDups.sort((a, b) => b[1] - a[1]).slice(0, 10);
366
+ console.log("\n Top 10 精确重复:");
367
+ for (const [text, count] of topDups) {
368
+ console.log(` [×${count}] ${text.slice(0, 80)}...`);
369
+ }
370
+ }
371
+
372
+ // Text fingerprint duplicates (first 100 chars match)
373
+ const fpMap = new Map<string, number>();
374
+ for (const row of parsedRows) {
375
+ const fp = row.text.trim().slice(0, 100).toLowerCase();
376
+ fpMap.set(fp, (fpMap.get(fp) || 0) + 1);
377
+ }
378
+ const fpDups = [...fpMap.entries()].filter(([, count]) => count > 1);
379
+ const fpDupRows = fpDups.reduce((s, [, c]) => s + c, 0);
380
+ console.log(`\n 前100字符重复组: ${fpDups.length}`);
381
+ console.log(` 相关行总数: ${fpDupRows} / ${total} (${(fpDupRows/total*100).toFixed(1)}%)`);
382
+
383
+ // ============================================
384
+ // 9. Scope 生态分析
385
+ // ============================================
386
+ console.log("\n📊 9. Scope 生态分析");
387
+ console.log("-".repeat(70));
388
+
389
+ const scopeCounts = new Map<string, number>();
390
+ for (const row of parsedRows) {
391
+ const s = row.scope || "(empty)";
392
+ scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
393
+ }
394
+ const totalScopes = scopeCounts.size;
395
+ const singletonScopes = [...scopeCounts.entries()].filter(([, c]) => c === 1).length;
396
+ const tinyScopes = [...scopeCounts.entries()].filter(([, c]) => c <= 5 && c > 1).length;
397
+
398
+ console.log(` 总 scope 数: ${totalScopes}`);
399
+ console.log(` 单条 scope(孤儿): ${singletonScopes} (${(singletonScopes/totalScopes*100).toFixed(1)}%)`);
400
+ console.log(` 2-5 条 scope: ${tinyScopes}`);
401
+ console.log(` Top 1 scope 占比: ${(([...scopeCounts.values()].sort((a,b)=>b-a)[0] / total) * 100).toFixed(1)}%`);
402
+
403
+ // ============================================
404
+ // 10. 内容主题聚类(基于 embedding 采样)
405
+ // ============================================
406
+ const clusterSampleSize = 2000;
407
+ console.log(`\n📊 10. Embedding 聚类采样(${clusterSampleSize} 条,K-Means k=8)`);
408
+ console.log("-".repeat(70));
409
+ console.log(" ⏳ 读取向量数据...");
410
+
411
+ // Read vectors for sample
412
+ const sampleWithVectors: MemoryRow[] = await table
413
+ .query()
414
+ .select(["id", "text", "vector", "category", "scope", "timestamp"])
415
+ .limit(clusterSampleSize)
416
+ .toArray() as any;
417
+
418
+ // Simple K-Means
419
+ const K = 8;
420
+ const dim = sampleWithVectors[0]?.vector?.length || 0;
421
+ if (dim === 0) {
422
+ console.log(" ⚠️ 无向量数据,跳过聚类");
423
+ } else {
424
+ console.log(` 向量维度: ${dim}`);
425
+
426
+ // Initialize centroids randomly
427
+ const centroids: number[][] = [];
428
+ const usedSet = new Set<number>();
429
+ for (let i = 0; i < K; i++) {
430
+ let idx: number;
431
+ do { idx = Math.floor(Math.random() * sampleWithVectors.length); } while (usedSet.has(idx));
432
+ usedSet.add(idx);
433
+ centroids.push([...sampleWithVectors[idx].vector]);
434
+ }
435
+
436
+ // Run K-Means for 20 iterations
437
+ const assignments = new Array(sampleWithVectors.length).fill(0);
438
+ for (let iter = 0; iter < 20; iter++) {
439
+ // Assign
440
+ for (let i = 0; i < sampleWithVectors.length; i++) {
441
+ let bestK = 0, bestSim = -1;
442
+ for (let k = 0; k < K; k++) {
443
+ const sim = cosineSimilarity(sampleWithVectors[i].vector, centroids[k]);
444
+ if (sim > bestSim) { bestSim = sim; bestK = k; }
445
+ }
446
+ assignments[i] = bestK;
447
+ }
448
+ // Update centroids
449
+ for (let k = 0; k < K; k++) {
450
+ const members = sampleWithVectors.filter((_, i) => assignments[i] === k);
451
+ if (members.length === 0) continue;
452
+ for (let d = 0; d < dim; d++) {
453
+ centroids[k][d] = members.reduce((s, m) => s + m.vector[d], 0) / members.length;
454
+ }
455
+ }
456
+ }
457
+
458
+ // Analyze clusters
459
+ const clusters: Map<number, typeof sampleWithVectors> = new Map();
460
+ for (let i = 0; i < sampleWithVectors.length; i++) {
461
+ if (!clusters.has(assignments[i])) clusters.set(assignments[i], []);
462
+ clusters.get(assignments[i])!.push(sampleWithVectors[i]);
463
+ }
464
+
465
+ console.log(`\n 聚类结果 (K=${K}):`);
466
+ const sortedClusters = [...clusters.entries()].sort((a, b) => b[1].length - a[1].length);
467
+
468
+ for (const [k, members] of sortedClusters) {
469
+ console.log(`\n ── Cluster ${k} (${members.length} 条, ${(members.length/sampleWithVectors.length*100).toFixed(1)}%) ──`);
470
+
471
+ // Category distribution within cluster
472
+ const clCatMap = new Map<string, number>();
473
+ for (const m of members) {
474
+ clCatMap.set(m.category, (clCatMap.get(m.category) || 0) + 1);
475
+ }
476
+ const catStr = [...clCatMap.entries()]
477
+ .sort((a, b) => b[1] - a[1])
478
+ .map(([c, n]) => `${c}:${n}`)
479
+ .join(", ");
480
+ console.log(` 类别: ${catStr}`);
481
+
482
+ // Top scopes
483
+ const clScopeMap = new Map<string, number>();
484
+ for (const m of members) {
485
+ const s = m.scope || "(empty)";
486
+ clScopeMap.set(s, (clScopeMap.get(s) || 0) + 1);
487
+ }
488
+ const topClScopes = [...clScopeMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
489
+ console.log(` Top scope: ${topClScopes.map(([s, n]) => `${s.slice(0,30)}(${n})`).join(", ")}`);
490
+
491
+ // Top keywords
492
+ const clWordFreq = new Map<string, number>();
493
+ for (const m of members) {
494
+ const tokens = tokenize(m.text);
495
+ const seen = new Set<string>();
496
+ for (const t of tokens) {
497
+ if (!STOPWORDS.has(t) && t.length > 1 && !seen.has(t)) {
498
+ clWordFreq.set(t, (clWordFreq.get(t) || 0) + 1);
499
+ seen.add(t);
500
+ }
501
+ }
502
+ }
503
+ const topClWords = [...clWordFreq.entries()]
504
+ .sort((a, b) => b[1] - a[1])
505
+ .slice(0, 8)
506
+ .map(([w]) => w)
507
+ .join(", ");
508
+ console.log(` 关键词: ${topClWords}`);
509
+
510
+ // Sample texts
511
+ console.log(` 示例:`);
512
+ const samples = members.slice(0, 3);
513
+ for (const s of samples) {
514
+ console.log(` - ${s.text.slice(0, 100)}...`);
515
+ }
516
+ }
517
+ }
518
+
519
+ // ============================================
520
+ // 11. 异常值检测
521
+ // ============================================
522
+ console.log("\n📊 11. 异常值检测");
523
+ console.log("-".repeat(70));
524
+
525
+ // Empty or near-empty scope
526
+ const emptyScope = parsedRows.filter(r => !r.scope || r.scope.trim() === "").length;
527
+ console.log(` 空 scope: ${emptyScope} 条 (${(emptyScope/total*100).toFixed(1)}%)`);
528
+
529
+ // Suspiciously high importance + zero access
530
+ const highImpZeroAccess = parsedRows.filter(r => {
531
+ const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
532
+ return r.importance >= 0.8 && ac === 0;
533
+ }).length;
534
+ console.log(` 高 importance(≥0.8) + 0 访问: ${highImpZeroAccess} 条`);
535
+
536
+ // Very old memories that are still "working" tier
537
+ const oldWorking = parsedRows.filter(r => {
538
+ const tier = r.meta.tier ?? "unknown";
539
+ const agedays = (Date.now() - r.timestamp) / (1000*60*60*24);
540
+ return tier === "working" && agedays > 60;
541
+ }).length;
542
+ console.log(` >60天的 working tier 记忆: ${oldWorking} 条(应降级?)`);
543
+
544
+ // Memories with no metadata
545
+ const noMeta = parsedRows.filter(r => !r.metadata || r.metadata === "{}").length;
546
+ console.log(` 无 metadata: ${noMeta} 条`);
547
+
548
+ // ============================================
549
+ // 综合发现
550
+ // ============================================
551
+ console.log("\n" + "=".repeat(70));
552
+ console.log("🔬 盲扫综合发现");
553
+ console.log("=".repeat(70));
554
+
555
+ const findings: string[] = [];
556
+
557
+ if (exactDups.length > 0) {
558
+ findings.push(`📌 精确重复 ${exactDups.length} 组 / ${totalDupRows} 条 — 有清理空间`);
559
+ }
560
+ if (ultraShort.length > 0) {
561
+ findings.push(`📌 超短文本 (<20字符) ${ultraShort.length} 条 — 可能是垃圾数据`);
562
+ }
563
+ if (singletonScopes > totalScopes * 0.5) {
564
+ findings.push(`📌 ${singletonScopes}/${totalScopes} scope 只有 1 条记忆 — scope 碎片化严重`);
565
+ }
566
+ if (highImpZeroAccess > 100) {
567
+ findings.push(`📌 ${highImpZeroAccess} 条高重要性记忆从未被访问 — importance 标记可能虚高`);
568
+ }
569
+ if (mixed / total > 0.3) {
570
+ findings.push(`📌 ${(mixed/total*100).toFixed(1)}% 记忆中英混合 — 可能影响检索一致性`);
571
+ }
572
+ if (emptyScope > total * 0.1) {
573
+ findings.push(`📌 ${(emptyScope/total*100).toFixed(1)}% 记忆无 scope — 检索时无法按项目过滤`);
574
+ }
575
+
576
+ for (const f of findings) {
577
+ console.log(` ${f}`);
578
+ }
579
+
580
+ if (findings.length === 0) {
581
+ console.log(" ✅ 未发现明显异常模式");
582
+ }
583
+
584
+ console.log("\n" + "=".repeat(70));
585
+ console.log("📋 盲扫完成\n");
586
+ }
587
+
588
+ main().catch(console.error);