recallnest 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +1 -1
  3. package/README.md +30 -5
  4. package/README_CN.md +30 -5
  5. package/config.json +55 -0
  6. package/connectors/examples/email-adapter.ts +73 -0
  7. package/connectors/examples/logs-adapter.ts +103 -0
  8. package/connectors/examples/rss-adapter.ts +95 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +60 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/ingest-2026-04-13.log +87 -0
  14. package/logs/launchagent-stderr.log +0 -0
  15. package/logs/launchagent-stdout.log +0 -0
  16. package/package.json +1 -1
  17. package/src/cli.ts +30 -0
  18. package/src/connector-types.ts +57 -0
  19. package/src/conversation-importer.ts +25 -1
  20. package/src/data-checkup.ts +31 -1
  21. package/src/doctor.ts +29 -0
  22. package/src/ingest.ts +203 -0
  23. package/src/mcp-server.ts +2 -2
  24. package/src/obsidian-connector.ts +282 -0
  25. package/src/session-schema.ts +2 -2
  26. package/src/source-heartbeat.ts +108 -0
  27. package/.claude-plugin/CLAUDE.md +0 -29
  28. package/.claude-plugin/marketplace.json +0 -18
  29. package/.dockerignore +0 -11
  30. package/.github/workflows/ci.yml +0 -28
  31. package/.mcp.json +0 -9
  32. package/CHANGELOG.md +0 -51
  33. package/DEMO.md +0 -94
  34. package/Dockerfile +0 -20
  35. package/EVAL.md +0 -89
  36. package/FAILURES.md +0 -75
  37. package/OPERATIONS.md +0 -59
  38. package/ROADMAP.md +0 -245
  39. package/SKILL.md +0 -250
  40. package/assets/dashboard.png +0 -0
  41. package/assets/knowledge-graph.png +0 -0
  42. package/assets/recallnest-workbench.png +0 -0
  43. package/assets/screenshots/ui-full.png +0 -0
  44. package/assets/screenshots/ui-overview.png +0 -0
  45. package/assets/screenshots/ui-search.png +0 -0
  46. package/assets/ui/app.js +0 -1031
  47. package/assets/ui/index.html +0 -224
  48. package/assets/ui/styles.css +0 -781
  49. package/assets/wechat-qr.jpg +0 -0
  50. package/bun.lock +0 -337
  51. package/daily-reflections/2026-03-05.md +0 -15
  52. package/daily-reflections/2026-03-06.md +0 -15
  53. package/daily-reflections/2026-03-07.md +0 -7
  54. package/daily-reflections/2026-03-08.md +0 -34
  55. package/daily-reflections/2026-03-09.md +0 -35
  56. package/daily-reflections/2026-03-10.md +0 -36
  57. package/daily-reflections/2026-03-11.md +0 -35
  58. package/daily-reflections/2026-03-12.md +0 -36
  59. package/daily-reflections/2026-03-13.md +0 -35
  60. package/daily-reflections/2026-03-14.md +0 -35
  61. package/daily-reflections/2026-03-15.md +0 -35
  62. package/daily-reflections/2026-03-16.md +0 -41
  63. package/daily-reflections/2026-03-17.md +0 -10
  64. package/daily-reflections/2026-03-18.md +0 -10
  65. package/daily-reflections/2026-03-19.md +0 -10
  66. package/daily-reflections/2026-03-20.md +0 -10
  67. package/daily-reflections/2026-03-21.md +0 -10
  68. package/daily-reflections/2026-03-22.md +0 -10
  69. package/daily-reflections/2026-03-23.md +0 -10
  70. package/daily-reflections/2026-03-24.md +0 -10
  71. package/daily-reflections/2026-03-25.md +0 -32
  72. package/daily-reflections/2026-03-26.md +0 -37
  73. package/daily-reflections/2026-03-27.md +0 -35
  74. package/daily-reflections/2026-03-28.md +0 -35
  75. package/daily-reflections/2026-03-29.md +0 -35
  76. package/daily-reflections/2026-03-30.md +0 -36
  77. package/daily-reflections/2026-03-31.md +0 -35
  78. package/daily-reflections/2026-04-01.md +0 -35
  79. package/daily-reflections/2026-04-02.md +0 -35
  80. package/daily-reflections/2026-04-03.md +0 -37
  81. package/daily-reflections/2026-04-04.md +0 -37
  82. package/daily-reflections/2026-04-05.md +0 -35
  83. package/daily-reflections/2026-04-06.md +0 -36
  84. package/daily-reflections/2026-04-07.md +0 -35
  85. package/daily-reflections/2026-04-08.md +0 -35
  86. package/daily-reflections/2026-04-12.md +0 -35
  87. package/data/alias-map.example.json +0 -22
  88. package/docs/api-reference.md +0 -1291
  89. package/docs/architecture.md +0 -168
  90. package/docs/memory-boundary-contract.md +0 -226
  91. package/docs/memory-categories.md +0 -119
  92. package/docs/pipeline-audit-ultramemory.md +0 -177
  93. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  94. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  95. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  96. package/docs/plans/mcp-tiering-plan.md +0 -126
  97. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  98. package/docs/self-evolution.md +0 -198
  99. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  100. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  101. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  102. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  103. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  104. package/eval/continuity/results-2026-03-16.md +0 -117
  105. package/eval/continuity/results-2026-03-19.md +0 -453
  106. package/eval/continuity/runbook.md +0 -110
  107. package/eval/continuity/seed-cases.md +0 -54
  108. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  109. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  110. package/eval/reports/2026-03-06-baseline.md +0 -82
  111. package/eval/reports/query-expansion.md +0 -138
  112. package/glama.json +0 -6
  113. package/lm +0 -19
  114. package/plan-philosophy-upgrades.md +0 -363
  115. package/plan-product-upgrades.md +0 -155
  116. package/plan-v1.2-distributable.md +0 -105
  117. package/plan.md +0 -140
  118. package/research-memory-evolution.md +0 -440
  119. package/research-rl-agent-memory.md +0 -330
  120. package/scripts/archive-distilled-junk.ts +0 -116
  121. package/scripts/backfill-anchors.ts +0 -130
  122. package/scripts/backfill-l0l1l2.ts +0 -98
  123. package/scripts/blind-scan.ts +0 -588
  124. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  125. package/scripts/cleanup-step2-template-dups.ts +0 -113
  126. package/scripts/cleanup-step3-stale-content.ts +0 -149
  127. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  128. package/scripts/delete-distilled-junk.ts +0 -88
  129. package/scripts/diagnose-vector-search.ts +0 -132
  130. package/scripts/distill-facts.ts +0 -305
  131. package/scripts/health-check.ts +0 -343
  132. package/scripts/incremental-ingest.sh +0 -71
  133. package/scripts/migrate-language.ts +0 -75
  134. package/scripts/mlp-health-check.ts +0 -382
  135. package/scripts/pull-claude-desktop.ts +0 -515
  136. package/scripts/re-ingest-cc.sh +0 -66
  137. package/scripts/rebalance-memory-health.ts +0 -181
  138. package/scripts/run-desktop-ingest.sh +0 -11
  139. package/scripts/start-server.sh +0 -12
  140. package/scripts/verify-emotion-decay.ts +0 -148
  141. package/scripts/viz-dashboard.html +0 -423
  142. package/scripts/viz-export.ts +0 -703
  143. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  144. package/src/__tests__/activity-counter.test.ts +0 -96
  145. package/src/__tests__/admission-control.test.ts +0 -206
  146. package/src/__tests__/anchor-generator.test.ts +0 -67
  147. package/src/__tests__/auto-consolidation.test.ts +0 -168
  148. package/src/__tests__/auto-recall.test.ts +0 -131
  149. package/src/__tests__/babel-integration.test.ts +0 -139
  150. package/src/__tests__/batch-dedup.test.ts +0 -65
  151. package/src/__tests__/capture-engine.test.ts +0 -1144
  152. package/src/__tests__/capture-heuristic.test.ts +0 -171
  153. package/src/__tests__/cascade-forget.test.ts +0 -210
  154. package/src/__tests__/category-min-scores.test.ts +0 -33
  155. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  156. package/src/__tests__/circuit-breaker.test.ts +0 -99
  157. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  158. package/src/__tests__/confidence-metadata.test.ts +0 -125
  159. package/src/__tests__/confidence-tracker.test.ts +0 -249
  160. package/src/__tests__/conflict-advisor.test.ts +0 -206
  161. package/src/__tests__/conflict-escalation.test.ts +0 -149
  162. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  163. package/src/__tests__/conflict-output.test.ts +0 -85
  164. package/src/__tests__/conflict-store.test.ts +0 -374
  165. package/src/__tests__/consolidation-engine.test.ts +0 -181
  166. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  167. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  168. package/src/__tests__/context-composer-pins.test.ts +0 -64
  169. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  170. package/src/__tests__/context-reconstructor.test.ts +0 -65
  171. package/src/__tests__/context-renderer.test.ts +0 -91
  172. package/src/__tests__/continuity-eval.test.ts +0 -275
  173. package/src/__tests__/continuity-schema.test.ts +0 -97
  174. package/src/__tests__/conversation-importer.test.ts +0 -692
  175. package/src/__tests__/core-recall-eval.test.ts +0 -235
  176. package/src/__tests__/core-summary.test.ts +0 -109
  177. package/src/__tests__/data-checkup.test.ts +0 -225
  178. package/src/__tests__/decay-exemption.test.ts +0 -84
  179. package/src/__tests__/dedup-actions.test.ts +0 -58
  180. package/src/__tests__/dedup-guard.test.ts +0 -39
  181. package/src/__tests__/deterministic-id.test.ts +0 -39
  182. package/src/__tests__/distill-lock.test.ts +0 -176
  183. package/src/__tests__/doctor.test.ts +0 -303
  184. package/src/__tests__/dream-pipeline.test.ts +0 -232
  185. package/src/__tests__/drill-down.test.ts +0 -108
  186. package/src/__tests__/embedder.test.ts +0 -122
  187. package/src/__tests__/emotion-decay.test.ts +0 -46
  188. package/src/__tests__/emotion-detector.test.ts +0 -56
  189. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  190. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  191. package/src/__tests__/entity-resolver.test.ts +0 -58
  192. package/src/__tests__/ethics-layer.test.ts +0 -476
  193. package/src/__tests__/event-segmenter.test.ts +0 -153
  194. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  195. package/src/__tests__/frequency-tracker.test.ts +0 -113
  196. package/src/__tests__/graph-export.test.ts +0 -489
  197. package/src/__tests__/hotness-blend.test.ts +0 -76
  198. package/src/__tests__/ingest.test.ts +0 -360
  199. package/src/__tests__/interference-detector.test.ts +0 -203
  200. package/src/__tests__/kg-extractor.test.ts +0 -184
  201. package/src/__tests__/kg-store.test.ts +0 -198
  202. package/src/__tests__/llm-consolidation.test.ts +0 -226
  203. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  204. package/src/__tests__/memory-boundaries.test.ts +0 -237
  205. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  206. package/src/__tests__/memory-evolution.test.ts +0 -271
  207. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  208. package/src/__tests__/memory-lint.test.ts +0 -271
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/narrative-architecture.test.ts +0 -456
  212. package/src/__tests__/pending-review.test.ts +0 -77
  213. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  214. package/src/__tests__/ppr-traversal.test.ts +0 -256
  215. package/src/__tests__/prediction-engine.test.ts +0 -349
  216. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  217. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  218. package/src/__tests__/preference-matcher.test.ts +0 -230
  219. package/src/__tests__/preference-slots.test.ts +0 -205
  220. package/src/__tests__/prospective-memory.test.ts +0 -283
  221. package/src/__tests__/query-entity-detector.test.ts +0 -171
  222. package/src/__tests__/query-expander-alias.test.ts +0 -89
  223. package/src/__tests__/query-expander.test.ts +0 -34
  224. package/src/__tests__/recall-governor.test.ts +0 -227
  225. package/src/__tests__/recall-mode.test.ts +0 -64
  226. package/src/__tests__/result-synthesizer.test.ts +0 -198
  227. package/src/__tests__/resume-context.test.ts +0 -4898
  228. package/src/__tests__/retrieval-trace.test.ts +0 -213
  229. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  230. package/src/__tests__/retroactive-boost.test.ts +0 -223
  231. package/src/__tests__/salience-scoring.test.ts +0 -106
  232. package/src/__tests__/scope-inventory.test.ts +0 -200
  233. package/src/__tests__/scope-policy.test.ts +0 -69
  234. package/src/__tests__/session-checkpoint.test.ts +0 -259
  235. package/src/__tests__/session-distiller.test.ts +0 -611
  236. package/src/__tests__/session-gc.test.ts +0 -215
  237. package/src/__tests__/skill-engine.test.ts +0 -546
  238. package/src/__tests__/skill-promotion.test.ts +0 -349
  239. package/src/__tests__/store-scope-audit.test.ts +0 -81
  240. package/src/__tests__/supersede-chain.test.ts +0 -154
  241. package/src/__tests__/temporal-parser.test.ts +0 -241
  242. package/src/__tests__/temporal-validity.test.ts +0 -160
  243. package/src/__tests__/term-registry.test.ts +0 -121
  244. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  245. package/src/__tests__/topic-tag.test.ts +0 -113
  246. package/src/__tests__/trusted-memory.test.ts +0 -828
  247. package/src/__tests__/version-manager.test.ts +0 -213
  248. package/src/__tests__/workflow-observation.test.ts +0 -400
  249. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,588 +0,0 @@
1
- /**
2
- * 盲扫脚本 — 模块 4:开放式探索
3
- * 不带预设地分析 38K 条记忆的文本内容
4
- * 只读,不写入任何内容
5
- *
6
- * 用法:bun run scripts/blind-scan.ts [lancedb-dir]
7
- */
8
-
9
- import lancedb from "@lancedb/lancedb";
10
-
11
- const DB_PATH = process.argv[2] || "./data/lancedb";
12
- const TABLE_NAME = "memories";
13
-
14
- interface MemoryRow {
15
- id: string;
16
- text: string;
17
- vector: number[];
18
- category: string;
19
- scope: string;
20
- importance: number;
21
- timestamp: number;
22
- metadata?: string;
23
- }
24
-
25
- function cosineSimilarity(a: number[], b: number[]): number {
26
- let dot = 0, normA = 0, normB = 0;
27
- for (let i = 0; i < a.length; i++) {
28
- dot += a[i] * b[i];
29
- normA += a[i] * a[i];
30
- normB += b[i] * b[i];
31
- }
32
- return dot / (Math.sqrt(normA) * Math.sqrt(normB));
33
- }
34
-
35
- function formatDate(ts: number): string {
36
- return new Date(ts).toISOString().slice(0, 10);
37
- }
38
-
39
- // Simple CJK-aware tokenizer
40
- function tokenize(text: string): string[] {
41
- const tokens: string[] = [];
42
- // Remove URLs
43
- const cleaned = text.replace(/https?:\/\/\S+/g, " ");
44
- // Split into words (English) and characters (CJK)
45
- const parts = cleaned.match(/[\u4e00-\u9fff\u3400-\u4dbf]+|[a-zA-Z][a-zA-Z0-9_'-]+/g) || [];
46
- for (const part of parts) {
47
- if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(part)) {
48
- // CJK: bigrams
49
- for (let i = 0; i < part.length - 1; i++) {
50
- tokens.push(part.slice(i, i + 2));
51
- }
52
- } else if (part.length > 1) {
53
- tokens.push(part.toLowerCase());
54
- }
55
- }
56
- return tokens;
57
- }
58
-
59
- // Stopwords (minimal)
60
- const STOPWORDS = new Set([
61
- "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
62
- "have", "has", "had", "do", "does", "did", "will", "would", "could",
63
- "should", "may", "might", "shall", "can", "need", "must",
64
- "to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
65
- "into", "through", "during", "before", "after", "above", "below",
66
- "between", "out", "off", "over", "under", "again", "further",
67
- "then", "once", "here", "there", "when", "where", "why", "how",
68
- "all", "each", "every", "both", "few", "more", "most", "other",
69
- "some", "such", "no", "nor", "not", "only", "own", "same", "so",
70
- "than", "too", "very", "just", "because", "but", "and", "or", "if",
71
- "while", "about", "up", "its", "it", "this", "that", "these", "those",
72
- "he", "she", "they", "we", "you", "me", "him", "her", "us", "them",
73
- "my", "your", "his", "our", "their", "what", "which", "who", "whom",
74
- "also", "been", "like", "get", "got", "make", "made", "use", "used",
75
- ]);
76
-
77
- async function main() {
78
- console.log("\n🔍 盲扫报告 — RecallNest 开放式探索");
79
- console.log("=".repeat(70));
80
- console.log(`📅 时间: ${new Date().toISOString().slice(0, 19)}`);
81
- console.log(`📂 路径: ${DB_PATH}\n`);
82
-
83
- const db = await lancedb.connect(DB_PATH);
84
- const table = await db.openTable(TABLE_NAME);
85
-
86
- console.log("⏳ 读取全量数据...");
87
- const allRows: MemoryRow[] = await table
88
- .query()
89
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
90
- .toArray() as any;
91
-
92
- const total = allRows.length;
93
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
94
-
95
- const parsedRows = allRows.map(row => ({
96
- ...row,
97
- meta: row.metadata ? JSON.parse(row.metadata) : {},
98
- }));
99
-
100
- // ============================================
101
- // 1. 文本长度分布
102
- // ============================================
103
- console.log("📊 1. 文本长度分布");
104
- console.log("-".repeat(70));
105
- const lengths = parsedRows.map(r => r.text.length);
106
- lengths.sort((a, b) => a - b);
107
- const avgLen = lengths.reduce((s, l) => s + l, 0) / total;
108
- const medianLen = lengths[Math.floor(total / 2)];
109
- const p95Len = lengths[Math.floor(total * 0.95)];
110
- const p99Len = lengths[Math.floor(total * 0.99)];
111
-
112
- console.log(` 最短: ${lengths[0]} 字符`);
113
- console.log(` 中位: ${medianLen} 字符`);
114
- console.log(` 平均: ${Math.round(avgLen)} 字符`);
115
- console.log(` P95: ${p95Len} 字符`);
116
- console.log(` P99: ${p99Len} 字符`);
117
- console.log(` 最长: ${lengths[total - 1]} 字符`);
118
-
119
- // Length buckets
120
- const lenBuckets = new Map<string, number>();
121
- const lenRanges: Array<[number, number, string]> = [
122
- [0, 50, "0-50"],
123
- [50, 100, "50-100"],
124
- [100, 200, "100-200"],
125
- [200, 500, "200-500"],
126
- [500, 1000, "500-1K"],
127
- [1000, 2000, "1K-2K"],
128
- [2000, 5000, "2K-5K"],
129
- [5000, Infinity, "5K+"],
130
- ];
131
- for (const len of lengths) {
132
- for (const [lo, hi, label] of lenRanges) {
133
- if (len >= lo && len < hi) {
134
- lenBuckets.set(label, (lenBuckets.get(label) || 0) + 1);
135
- break;
136
- }
137
- }
138
- }
139
- console.log("\n 分布:");
140
- for (const [, , label] of lenRanges) {
141
- const count = lenBuckets.get(label) || 0;
142
- const pct = (count / total * 100).toFixed(1);
143
- const bar = "█".repeat(Math.round(parseFloat(pct) / 100 * 40));
144
- console.log(` ${label.padEnd(10)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${bar}`);
145
- }
146
-
147
- // Extremely short texts (likely garbage)
148
- const ultraShort = parsedRows.filter(r => r.text.length < 20);
149
- if (ultraShort.length > 0) {
150
- console.log(`\n ⚠️ 超短文本 (<20 字符): ${ultraShort.length} 条`);
151
- console.log(" 示例:");
152
- for (const r of ultraShort.slice(0, 5)) {
153
- console.log(` [${r.category}] "${r.text}"`);
154
- }
155
- }
156
-
157
- // ============================================
158
- // 2. 语言混合分析
159
- // ============================================
160
- console.log("\n📊 2. 语言混合分析");
161
- console.log("-".repeat(70));
162
- let pureEn = 0, pureCn = 0, mixed = 0, other = 0;
163
- for (const row of parsedRows) {
164
- const hasCJK = /[\u4e00-\u9fff]/.test(row.text);
165
- const hasEn = /[a-zA-Z]{3,}/.test(row.text);
166
- if (hasCJK && hasEn) mixed++;
167
- else if (hasCJK) pureCn++;
168
- else if (hasEn) pureEn++;
169
- else other++;
170
- }
171
- console.log(` 纯英文: ${pureEn.toLocaleString()} (${(pureEn/total*100).toFixed(1)}%)`);
172
- console.log(` 纯中文: ${pureCn.toLocaleString()} (${(pureCn/total*100).toFixed(1)}%)`);
173
- console.log(` 中英混合: ${mixed.toLocaleString()} (${(mixed/total*100).toFixed(1)}%)`);
174
- console.log(` 其他: ${other.toLocaleString()} (${(other/total*100).toFixed(1)}%)`);
175
-
176
- // ============================================
177
- // 3. 高频词分析(分语言)
178
- // ============================================
179
- console.log("\n📊 3. 高频词分析");
180
- console.log("-".repeat(70));
181
- const wordFreq = new Map<string, number>();
182
- for (const row of parsedRows) {
183
- const tokens = tokenize(row.text);
184
- for (const t of tokens) {
185
- if (!STOPWORDS.has(t) && t.length > 1) {
186
- wordFreq.set(t, (wordFreq.get(t) || 0) + 1);
187
- }
188
- }
189
- }
190
- const topWords = [...wordFreq.entries()]
191
- .sort((a, b) => b[1] - a[1])
192
- .slice(0, 40);
193
-
194
- console.log(" Top 40 词频(去停用词):");
195
- for (let i = 0; i < topWords.length; i++) {
196
- const [word, count] = topWords[i];
197
- const pctOfDocs = (count / total * 100).toFixed(1);
198
- console.log(` ${(i+1).toString().padStart(2)}. ${word.padEnd(20)} ${count.toString().padStart(6)} (${pctOfDocs}% 文档)`);
199
- }
200
-
201
- // ============================================
202
- // 4. 文本前缀聚类(发现模板化内容)
203
- // ============================================
204
- console.log("\n📊 4. 文本前缀聚类(发现模板化内容)");
205
- console.log("-".repeat(70));
206
- const prefixMap = new Map<string, number>();
207
- for (const row of parsedRows) {
208
- // Take first 40 chars as prefix fingerprint
209
- const prefix = row.text.slice(0, 40).trim();
210
- prefixMap.set(prefix, (prefixMap.get(prefix) || 0) + 1);
211
- }
212
- const repeatedPrefixes = [...prefixMap.entries()]
213
- .filter(([, count]) => count >= 3)
214
- .sort((a, b) => b[1] - a[1]);
215
-
216
- console.log(` 独立前缀数: ${prefixMap.size.toLocaleString()}`);
217
- console.log(` 重复 ≥3 次的前缀: ${repeatedPrefixes.length}`);
218
-
219
- if (repeatedPrefixes.length > 0) {
220
- console.log("\n Top 20 重复前缀:");
221
- for (const [prefix, count] of repeatedPrefixes.slice(0, 20)) {
222
- console.log(` [×${count.toString().padStart(4)}] ${prefix}...`);
223
- }
224
- }
225
-
226
- // ============================================
227
- // 5. Scope × Category 交叉分析
228
- // ============================================
229
- console.log("\n📊 5. Scope × Category 交叉分析(Top 10 Scope)");
230
- console.log("-".repeat(70));
231
- const scopeCat = new Map<string, Map<string, number>>();
232
- for (const row of parsedRows) {
233
- const s = row.scope || "(empty)";
234
- if (!scopeCat.has(s)) scopeCat.set(s, new Map());
235
- const catMap = scopeCat.get(s)!;
236
- catMap.set(row.category, (catMap.get(row.category) || 0) + 1);
237
- }
238
- const topScopes = [...scopeCat.entries()]
239
- .map(([scope, cats]) => {
240
- const scopeTotal = [...cats.values()].reduce((s, c) => s + c, 0);
241
- return { scope, total: scopeTotal, cats };
242
- })
243
- .sort((a, b) => b.total - a.total)
244
- .slice(0, 10);
245
-
246
- const allCats = [...new Set(parsedRows.map(r => r.category))].sort();
247
- // Header
248
- let header = " Scope".padEnd(35);
249
- for (const cat of allCats) header += cat.slice(0, 6).padStart(8);
250
- header += " Total".padStart(8);
251
- console.log(header);
252
-
253
- for (const { scope, total: scopeTotal, cats } of topScopes) {
254
- const label = scope.length > 30 ? scope.slice(0, 27) + "..." : scope;
255
- let line = ` ${label.padEnd(33)}`;
256
- for (const cat of allCats) {
257
- line += (cats.get(cat) || 0).toString().padStart(8);
258
- }
259
- line += scopeTotal.toString().padStart(8);
260
- console.log(line);
261
- }
262
-
263
- // ============================================
264
- // 6. 时间模式分析
265
- // ============================================
266
- console.log("\n📊 6. 时间模式分析");
267
- console.log("-".repeat(70));
268
-
269
- // By month
270
- const monthMap = new Map<string, number>();
271
- // By hour of day
272
- const hourMap = new Map<number, number>();
273
- // By day of week
274
- const dowMap = new Map<number, number>();
275
- const DOW_NAMES = ["周日", "周一", "周二", "周三", "周四", "周五", "周六"];
276
-
277
- for (const row of parsedRows) {
278
- const d = new Date(row.timestamp);
279
- const month = d.toISOString().slice(0, 7);
280
- monthMap.set(month, (monthMap.get(month) || 0) + 1);
281
- hourMap.set(d.getHours(), (hourMap.get(d.getHours()) || 0) + 1);
282
- dowMap.set(d.getDay(), (dowMap.get(d.getDay()) || 0) + 1);
283
- }
284
-
285
- // Monthly trend
286
- const months = [...monthMap.entries()].sort();
287
- console.log(" 月度趋势:");
288
- const maxMonthCount = Math.max(...months.map(m => m[1]));
289
- for (const [month, count] of months) {
290
- const bar = "█".repeat(Math.round(count / maxMonthCount * 30));
291
- console.log(` ${month} ${count.toString().padStart(6)} ${bar}`);
292
- }
293
-
294
- // Hourly pattern
295
- console.log("\n 小时分布:");
296
- const maxHourCount = Math.max(...[...hourMap.values()]);
297
- for (let h = 0; h < 24; h++) {
298
- const count = hourMap.get(h) || 0;
299
- const bar = "█".repeat(Math.round(count / maxHourCount * 20));
300
- console.log(` ${h.toString().padStart(2)}:00 ${count.toString().padStart(5)} ${bar}`);
301
- }
302
-
303
- // Day of week
304
- console.log("\n 星期分布:");
305
- for (let d = 0; d < 7; d++) {
306
- const count = dowMap.get(d) || 0;
307
- const pct = (count / total * 100).toFixed(1);
308
- console.log(` ${DOW_NAMES[d]} ${count.toString().padStart(6)} (${pct}%)`);
309
- }
310
-
311
- // ============================================
312
- // 7. 过时内容检测
313
- // ============================================
314
- console.log("\n📊 7. 过时内容检测");
315
- console.log("-".repeat(70));
316
-
317
- // Look for date references in text
318
- const yearRefs = new Map<string, number>();
319
- for (const row of parsedRows) {
320
- const yearPattern = /20(2[0-6])/g;
321
- let match;
322
- while ((match = yearPattern.exec(row.text)) !== null) {
323
- const year = `20${match[1]}`;
324
- yearRefs.set(year, (yearRefs.get(year) || 0) + 1);
325
- }
326
- }
327
- console.log(" 文本中提到的年份:");
328
- for (const [year, count] of [...yearRefs.entries()].sort()) {
329
- console.log(` ${year}: ${count.toLocaleString()} 次`);
330
- }
331
-
332
- // Look for potentially outdated keywords
333
- const outdatedKeywords = [
334
- "deprecated", "removed", "old", "legacy", "TODO", "FIXME", "HACK",
335
- "temporary", "workaround", "临时", "废弃", "已删除", "待修复",
336
- ];
337
- console.log("\n 过时关键词扫描:");
338
- for (const kw of outdatedKeywords) {
339
- const count = parsedRows.filter(r =>
340
- r.text.toLowerCase().includes(kw.toLowerCase())
341
- ).length;
342
- if (count > 0) {
343
- console.log(` "${kw}": ${count} 条`);
344
- }
345
- }
346
-
347
- // ============================================
348
- // 8. 文本精确重复检测
349
- // ============================================
350
- console.log("\n📊 8. 精确/近似文本重复");
351
- console.log("-".repeat(70));
352
-
353
- // Exact text duplicates
354
- const textMap = new Map<string, number>();
355
- for (const row of parsedRows) {
356
- const normalized = row.text.trim().toLowerCase();
357
- textMap.set(normalized, (textMap.get(normalized) || 0) + 1);
358
- }
359
- const exactDups = [...textMap.entries()].filter(([, count]) => count > 1);
360
- const totalDupRows = exactDups.reduce((s, [, c]) => s + c, 0);
361
- console.log(` 精确重复组: ${exactDups.length}`);
362
- console.log(` 重复行总数: ${totalDupRows} / ${total} (${(totalDupRows/total*100).toFixed(1)}%)`);
363
-
364
- if (exactDups.length > 0) {
365
- const topDups = exactDups.sort((a, b) => b[1] - a[1]).slice(0, 10);
366
- console.log("\n Top 10 精确重复:");
367
- for (const [text, count] of topDups) {
368
- console.log(` [×${count}] ${text.slice(0, 80)}...`);
369
- }
370
- }
371
-
372
- // Text fingerprint duplicates (first 100 chars match)
373
- const fpMap = new Map<string, number>();
374
- for (const row of parsedRows) {
375
- const fp = row.text.trim().slice(0, 100).toLowerCase();
376
- fpMap.set(fp, (fpMap.get(fp) || 0) + 1);
377
- }
378
- const fpDups = [...fpMap.entries()].filter(([, count]) => count > 1);
379
- const fpDupRows = fpDups.reduce((s, [, c]) => s + c, 0);
380
- console.log(`\n 前100字符重复组: ${fpDups.length}`);
381
- console.log(` 相关行总数: ${fpDupRows} / ${total} (${(fpDupRows/total*100).toFixed(1)}%)`);
382
-
383
- // ============================================
384
- // 9. Scope 生态分析
385
- // ============================================
386
- console.log("\n📊 9. Scope 生态分析");
387
- console.log("-".repeat(70));
388
-
389
- const scopeCounts = new Map<string, number>();
390
- for (const row of parsedRows) {
391
- const s = row.scope || "(empty)";
392
- scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
393
- }
394
- const totalScopes = scopeCounts.size;
395
- const singletonScopes = [...scopeCounts.entries()].filter(([, c]) => c === 1).length;
396
- const tinyScopes = [...scopeCounts.entries()].filter(([, c]) => c <= 5 && c > 1).length;
397
-
398
- console.log(` 总 scope 数: ${totalScopes}`);
399
- console.log(` 单条 scope(孤儿): ${singletonScopes} (${(singletonScopes/totalScopes*100).toFixed(1)}%)`);
400
- console.log(` 2-5 条 scope: ${tinyScopes}`);
401
- console.log(` Top 1 scope 占比: ${(([...scopeCounts.values()].sort((a,b)=>b-a)[0] / total) * 100).toFixed(1)}%`);
402
-
403
- // ============================================
404
- // 10. 内容主题聚类(基于 embedding 采样)
405
- // ============================================
406
- const clusterSampleSize = 2000;
407
- console.log(`\n📊 10. Embedding 聚类采样(${clusterSampleSize} 条,K-Means k=8)`);
408
- console.log("-".repeat(70));
409
- console.log(" ⏳ 读取向量数据...");
410
-
411
- // Read vectors for sample
412
- const sampleWithVectors: MemoryRow[] = await table
413
- .query()
414
- .select(["id", "text", "vector", "category", "scope", "timestamp"])
415
- .limit(clusterSampleSize)
416
- .toArray() as any;
417
-
418
- // Simple K-Means
419
- const K = 8;
420
- const dim = sampleWithVectors[0]?.vector?.length || 0;
421
- if (dim === 0) {
422
- console.log(" ⚠️ 无向量数据,跳过聚类");
423
- } else {
424
- console.log(` 向量维度: ${dim}`);
425
-
426
- // Initialize centroids randomly
427
- const centroids: number[][] = [];
428
- const usedSet = new Set<number>();
429
- for (let i = 0; i < K; i++) {
430
- let idx: number;
431
- do { idx = Math.floor(Math.random() * sampleWithVectors.length); } while (usedSet.has(idx));
432
- usedSet.add(idx);
433
- centroids.push([...sampleWithVectors[idx].vector]);
434
- }
435
-
436
- // Run K-Means for 20 iterations
437
- const assignments = new Array(sampleWithVectors.length).fill(0);
438
- for (let iter = 0; iter < 20; iter++) {
439
- // Assign
440
- for (let i = 0; i < sampleWithVectors.length; i++) {
441
- let bestK = 0, bestSim = -1;
442
- for (let k = 0; k < K; k++) {
443
- const sim = cosineSimilarity(sampleWithVectors[i].vector, centroids[k]);
444
- if (sim > bestSim) { bestSim = sim; bestK = k; }
445
- }
446
- assignments[i] = bestK;
447
- }
448
- // Update centroids
449
- for (let k = 0; k < K; k++) {
450
- const members = sampleWithVectors.filter((_, i) => assignments[i] === k);
451
- if (members.length === 0) continue;
452
- for (let d = 0; d < dim; d++) {
453
- centroids[k][d] = members.reduce((s, m) => s + m.vector[d], 0) / members.length;
454
- }
455
- }
456
- }
457
-
458
- // Analyze clusters
459
- const clusters: Map<number, typeof sampleWithVectors> = new Map();
460
- for (let i = 0; i < sampleWithVectors.length; i++) {
461
- if (!clusters.has(assignments[i])) clusters.set(assignments[i], []);
462
- clusters.get(assignments[i])!.push(sampleWithVectors[i]);
463
- }
464
-
465
- console.log(`\n 聚类结果 (K=${K}):`);
466
- const sortedClusters = [...clusters.entries()].sort((a, b) => b[1].length - a[1].length);
467
-
468
- for (const [k, members] of sortedClusters) {
469
- console.log(`\n ── Cluster ${k} (${members.length} 条, ${(members.length/sampleWithVectors.length*100).toFixed(1)}%) ──`);
470
-
471
- // Category distribution within cluster
472
- const clCatMap = new Map<string, number>();
473
- for (const m of members) {
474
- clCatMap.set(m.category, (clCatMap.get(m.category) || 0) + 1);
475
- }
476
- const catStr = [...clCatMap.entries()]
477
- .sort((a, b) => b[1] - a[1])
478
- .map(([c, n]) => `${c}:${n}`)
479
- .join(", ");
480
- console.log(` 类别: ${catStr}`);
481
-
482
- // Top scopes
483
- const clScopeMap = new Map<string, number>();
484
- for (const m of members) {
485
- const s = m.scope || "(empty)";
486
- clScopeMap.set(s, (clScopeMap.get(s) || 0) + 1);
487
- }
488
- const topClScopes = [...clScopeMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
489
- console.log(` Top scope: ${topClScopes.map(([s, n]) => `${s.slice(0,30)}(${n})`).join(", ")}`);
490
-
491
- // Top keywords
492
- const clWordFreq = new Map<string, number>();
493
- for (const m of members) {
494
- const tokens = tokenize(m.text);
495
- const seen = new Set<string>();
496
- for (const t of tokens) {
497
- if (!STOPWORDS.has(t) && t.length > 1 && !seen.has(t)) {
498
- clWordFreq.set(t, (clWordFreq.get(t) || 0) + 1);
499
- seen.add(t);
500
- }
501
- }
502
- }
503
- const topClWords = [...clWordFreq.entries()]
504
- .sort((a, b) => b[1] - a[1])
505
- .slice(0, 8)
506
- .map(([w]) => w)
507
- .join(", ");
508
- console.log(` 关键词: ${topClWords}`);
509
-
510
- // Sample texts
511
- console.log(` 示例:`);
512
- const samples = members.slice(0, 3);
513
- for (const s of samples) {
514
- console.log(` - ${s.text.slice(0, 100)}...`);
515
- }
516
- }
517
- }
518
-
519
- // ============================================
520
- // 11. 异常值检测
521
- // ============================================
522
- console.log("\n📊 11. 异常值检测");
523
- console.log("-".repeat(70));
524
-
525
- // Empty or near-empty scope
526
- const emptyScope = parsedRows.filter(r => !r.scope || r.scope.trim() === "").length;
527
- console.log(` 空 scope: ${emptyScope} 条 (${(emptyScope/total*100).toFixed(1)}%)`);
528
-
529
- // Suspiciously high importance + zero access
530
- const highImpZeroAccess = parsedRows.filter(r => {
531
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
532
- return r.importance >= 0.8 && ac === 0;
533
- }).length;
534
- console.log(` 高 importance(≥0.8) + 0 访问: ${highImpZeroAccess} 条`);
535
-
536
- // Very old memories that are still "working" tier
537
- const oldWorking = parsedRows.filter(r => {
538
- const tier = r.meta.tier ?? "unknown";
539
- const agedays = (Date.now() - r.timestamp) / (1000*60*60*24);
540
- return tier === "working" && agedays > 60;
541
- }).length;
542
- console.log(` >60天的 working tier 记忆: ${oldWorking} 条(应降级?)`);
543
-
544
- // Memories with no metadata
545
- const noMeta = parsedRows.filter(r => !r.metadata || r.metadata === "{}").length;
546
- console.log(` 无 metadata: ${noMeta} 条`);
547
-
548
- // ============================================
549
- // 综合发现
550
- // ============================================
551
- console.log("\n" + "=".repeat(70));
552
- console.log("🔬 盲扫综合发现");
553
- console.log("=".repeat(70));
554
-
555
- const findings: string[] = [];
556
-
557
- if (exactDups.length > 0) {
558
- findings.push(`📌 精确重复 ${exactDups.length} 组 / ${totalDupRows} 条 — 有清理空间`);
559
- }
560
- if (ultraShort.length > 0) {
561
- findings.push(`📌 超短文本 (<20字符) ${ultraShort.length} 条 — 可能是垃圾数据`);
562
- }
563
- if (singletonScopes > totalScopes * 0.5) {
564
- findings.push(`📌 ${singletonScopes}/${totalScopes} scope 只有 1 条记忆 — scope 碎片化严重`);
565
- }
566
- if (highImpZeroAccess > 100) {
567
- findings.push(`📌 ${highImpZeroAccess} 条高重要性记忆从未被访问 — importance 标记可能虚高`);
568
- }
569
- if (mixed / total > 0.3) {
570
- findings.push(`📌 ${(mixed/total*100).toFixed(1)}% 记忆中英混合 — 可能影响检索一致性`);
571
- }
572
- if (emptyScope > total * 0.1) {
573
- findings.push(`📌 ${(emptyScope/total*100).toFixed(1)}% 记忆无 scope — 检索时无法按项目过滤`);
574
- }
575
-
576
- for (const f of findings) {
577
- console.log(` ${f}`);
578
- }
579
-
580
- if (findings.length === 0) {
581
- console.log(" ✅ 未发现明显异常模式");
582
- }
583
-
584
- console.log("\n" + "=".repeat(70));
585
- console.log("📋 盲扫完成\n");
586
- }
587
-
588
- main().catch(console.error);