recallnest 2.2.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/.env +4 -0
  2. package/config.json +55 -0
  3. package/logs/daily-recall.error.log +0 -0
  4. package/logs/daily-recall.log +30 -0
  5. package/logs/ingest-2026-04-11.log +110 -0
  6. package/logs/ingest-2026-04-12.log +90 -0
  7. package/logs/launchagent-stderr.log +0 -0
  8. package/logs/launchagent-stdout.log +0 -0
  9. package/package.json +1 -1
  10. package/.claude-plugin/CLAUDE.md +0 -29
  11. package/.claude-plugin/marketplace.json +0 -18
  12. package/.dockerignore +0 -11
  13. package/.github/workflows/ci.yml +0 -28
  14. package/.mcp.json +0 -9
  15. package/CHANGELOG.md +0 -51
  16. package/DEMO.md +0 -94
  17. package/Dockerfile +0 -20
  18. package/EVAL.md +0 -89
  19. package/FAILURES.md +0 -75
  20. package/OPERATIONS.md +0 -59
  21. package/ROADMAP.md +0 -245
  22. package/SKILL.md +0 -250
  23. package/assets/dashboard.png +0 -0
  24. package/assets/knowledge-graph.png +0 -0
  25. package/assets/recallnest-workbench.png +0 -0
  26. package/assets/screenshots/ui-full.png +0 -0
  27. package/assets/screenshots/ui-overview.png +0 -0
  28. package/assets/screenshots/ui-search.png +0 -0
  29. package/assets/ui/app.js +0 -1031
  30. package/assets/ui/index.html +0 -224
  31. package/assets/ui/styles.css +0 -781
  32. package/assets/wechat-qr.jpg +0 -0
  33. package/bun.lock +0 -337
  34. package/daily-reflections/2026-03-05.md +0 -15
  35. package/daily-reflections/2026-03-06.md +0 -15
  36. package/daily-reflections/2026-03-07.md +0 -7
  37. package/daily-reflections/2026-03-08.md +0 -34
  38. package/daily-reflections/2026-03-09.md +0 -35
  39. package/daily-reflections/2026-03-10.md +0 -36
  40. package/daily-reflections/2026-03-11.md +0 -35
  41. package/daily-reflections/2026-03-12.md +0 -36
  42. package/daily-reflections/2026-03-13.md +0 -35
  43. package/daily-reflections/2026-03-14.md +0 -35
  44. package/daily-reflections/2026-03-15.md +0 -35
  45. package/daily-reflections/2026-03-16.md +0 -41
  46. package/daily-reflections/2026-03-17.md +0 -10
  47. package/daily-reflections/2026-03-18.md +0 -10
  48. package/daily-reflections/2026-03-19.md +0 -10
  49. package/daily-reflections/2026-03-20.md +0 -10
  50. package/daily-reflections/2026-03-21.md +0 -10
  51. package/daily-reflections/2026-03-22.md +0 -10
  52. package/daily-reflections/2026-03-23.md +0 -10
  53. package/daily-reflections/2026-03-24.md +0 -10
  54. package/daily-reflections/2026-03-25.md +0 -32
  55. package/daily-reflections/2026-03-26.md +0 -37
  56. package/daily-reflections/2026-03-27.md +0 -35
  57. package/daily-reflections/2026-03-28.md +0 -35
  58. package/daily-reflections/2026-03-29.md +0 -35
  59. package/daily-reflections/2026-03-30.md +0 -36
  60. package/daily-reflections/2026-03-31.md +0 -35
  61. package/daily-reflections/2026-04-01.md +0 -35
  62. package/daily-reflections/2026-04-02.md +0 -35
  63. package/daily-reflections/2026-04-03.md +0 -37
  64. package/daily-reflections/2026-04-04.md +0 -37
  65. package/daily-reflections/2026-04-05.md +0 -35
  66. package/daily-reflections/2026-04-06.md +0 -36
  67. package/daily-reflections/2026-04-07.md +0 -35
  68. package/daily-reflections/2026-04-08.md +0 -35
  69. package/daily-reflections/2026-04-12.md +0 -35
  70. package/data/alias-map.example.json +0 -22
  71. package/docs/api-reference.md +0 -1291
  72. package/docs/architecture.md +0 -168
  73. package/docs/memory-boundary-contract.md +0 -226
  74. package/docs/memory-categories.md +0 -119
  75. package/docs/pipeline-audit-ultramemory.md +0 -177
  76. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  77. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  78. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  79. package/docs/plans/mcp-tiering-plan.md +0 -126
  80. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  81. package/docs/self-evolution.md +0 -198
  82. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  83. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  84. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  85. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  86. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  87. package/eval/continuity/results-2026-03-16.md +0 -117
  88. package/eval/continuity/results-2026-03-19.md +0 -453
  89. package/eval/continuity/runbook.md +0 -110
  90. package/eval/continuity/seed-cases.md +0 -54
  91. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  92. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  93. package/eval/reports/2026-03-06-baseline.md +0 -82
  94. package/eval/reports/query-expansion.md +0 -138
  95. package/glama.json +0 -6
  96. package/lm +0 -19
  97. package/plan-philosophy-upgrades.md +0 -363
  98. package/plan-product-upgrades.md +0 -155
  99. package/plan-v1.2-distributable.md +0 -105
  100. package/plan.md +0 -140
  101. package/research-memory-evolution.md +0 -440
  102. package/research-rl-agent-memory.md +0 -330
  103. package/scripts/archive-distilled-junk.ts +0 -116
  104. package/scripts/backfill-anchors.ts +0 -130
  105. package/scripts/backfill-l0l1l2.ts +0 -98
  106. package/scripts/blind-scan.ts +0 -588
  107. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  108. package/scripts/cleanup-step2-template-dups.ts +0 -113
  109. package/scripts/cleanup-step3-stale-content.ts +0 -149
  110. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  111. package/scripts/delete-distilled-junk.ts +0 -88
  112. package/scripts/diagnose-vector-search.ts +0 -132
  113. package/scripts/distill-facts.ts +0 -305
  114. package/scripts/health-check.ts +0 -343
  115. package/scripts/incremental-ingest.sh +0 -71
  116. package/scripts/migrate-language.ts +0 -75
  117. package/scripts/mlp-health-check.ts +0 -382
  118. package/scripts/pull-claude-desktop.ts +0 -515
  119. package/scripts/re-ingest-cc.sh +0 -66
  120. package/scripts/rebalance-memory-health.ts +0 -181
  121. package/scripts/run-desktop-ingest.sh +0 -11
  122. package/scripts/start-server.sh +0 -12
  123. package/scripts/verify-emotion-decay.ts +0 -148
  124. package/scripts/viz-dashboard.html +0 -423
  125. package/scripts/viz-export.ts +0 -703
  126. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  127. package/src/__tests__/activity-counter.test.ts +0 -96
  128. package/src/__tests__/admission-control.test.ts +0 -206
  129. package/src/__tests__/anchor-generator.test.ts +0 -67
  130. package/src/__tests__/auto-consolidation.test.ts +0 -168
  131. package/src/__tests__/auto-recall.test.ts +0 -131
  132. package/src/__tests__/babel-integration.test.ts +0 -139
  133. package/src/__tests__/batch-dedup.test.ts +0 -65
  134. package/src/__tests__/capture-engine.test.ts +0 -1144
  135. package/src/__tests__/capture-heuristic.test.ts +0 -171
  136. package/src/__tests__/cascade-forget.test.ts +0 -210
  137. package/src/__tests__/category-min-scores.test.ts +0 -33
  138. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  139. package/src/__tests__/circuit-breaker.test.ts +0 -99
  140. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  141. package/src/__tests__/confidence-metadata.test.ts +0 -125
  142. package/src/__tests__/confidence-tracker.test.ts +0 -249
  143. package/src/__tests__/conflict-advisor.test.ts +0 -206
  144. package/src/__tests__/conflict-escalation.test.ts +0 -149
  145. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  146. package/src/__tests__/conflict-output.test.ts +0 -85
  147. package/src/__tests__/conflict-store.test.ts +0 -374
  148. package/src/__tests__/consolidation-engine.test.ts +0 -181
  149. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  150. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  151. package/src/__tests__/context-composer-pins.test.ts +0 -64
  152. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  153. package/src/__tests__/context-reconstructor.test.ts +0 -65
  154. package/src/__tests__/context-renderer.test.ts +0 -91
  155. package/src/__tests__/continuity-eval.test.ts +0 -275
  156. package/src/__tests__/continuity-schema.test.ts +0 -97
  157. package/src/__tests__/conversation-importer.test.ts +0 -692
  158. package/src/__tests__/core-recall-eval.test.ts +0 -235
  159. package/src/__tests__/core-summary.test.ts +0 -109
  160. package/src/__tests__/data-checkup.test.ts +0 -225
  161. package/src/__tests__/decay-exemption.test.ts +0 -84
  162. package/src/__tests__/dedup-actions.test.ts +0 -58
  163. package/src/__tests__/dedup-guard.test.ts +0 -39
  164. package/src/__tests__/deterministic-id.test.ts +0 -39
  165. package/src/__tests__/distill-lock.test.ts +0 -176
  166. package/src/__tests__/doctor.test.ts +0 -303
  167. package/src/__tests__/dream-pipeline.test.ts +0 -232
  168. package/src/__tests__/drill-down.test.ts +0 -108
  169. package/src/__tests__/embedder.test.ts +0 -122
  170. package/src/__tests__/emotion-decay.test.ts +0 -46
  171. package/src/__tests__/emotion-detector.test.ts +0 -56
  172. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  173. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  174. package/src/__tests__/entity-resolver.test.ts +0 -58
  175. package/src/__tests__/ethics-layer.test.ts +0 -476
  176. package/src/__tests__/event-segmenter.test.ts +0 -153
  177. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  178. package/src/__tests__/frequency-tracker.test.ts +0 -113
  179. package/src/__tests__/graph-export.test.ts +0 -489
  180. package/src/__tests__/hotness-blend.test.ts +0 -76
  181. package/src/__tests__/ingest.test.ts +0 -360
  182. package/src/__tests__/interference-detector.test.ts +0 -203
  183. package/src/__tests__/kg-extractor.test.ts +0 -184
  184. package/src/__tests__/kg-store.test.ts +0 -198
  185. package/src/__tests__/llm-consolidation.test.ts +0 -226
  186. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  187. package/src/__tests__/memory-boundaries.test.ts +0 -237
  188. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  189. package/src/__tests__/memory-evolution.test.ts +0 -271
  190. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  191. package/src/__tests__/memory-lint.test.ts +0 -271
  192. package/src/__tests__/memory-output.test.ts +0 -215
  193. package/src/__tests__/multi-vector.test.ts +0 -242
  194. package/src/__tests__/narrative-architecture.test.ts +0 -456
  195. package/src/__tests__/pending-review.test.ts +0 -77
  196. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  197. package/src/__tests__/ppr-traversal.test.ts +0 -256
  198. package/src/__tests__/prediction-engine.test.ts +0 -349
  199. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  200. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  201. package/src/__tests__/preference-matcher.test.ts +0 -230
  202. package/src/__tests__/preference-slots.test.ts +0 -205
  203. package/src/__tests__/prospective-memory.test.ts +0 -283
  204. package/src/__tests__/query-entity-detector.test.ts +0 -171
  205. package/src/__tests__/query-expander-alias.test.ts +0 -89
  206. package/src/__tests__/query-expander.test.ts +0 -34
  207. package/src/__tests__/recall-governor.test.ts +0 -227
  208. package/src/__tests__/recall-mode.test.ts +0 -64
  209. package/src/__tests__/result-synthesizer.test.ts +0 -198
  210. package/src/__tests__/resume-context.test.ts +0 -4898
  211. package/src/__tests__/retrieval-trace.test.ts +0 -213
  212. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  213. package/src/__tests__/retroactive-boost.test.ts +0 -223
  214. package/src/__tests__/salience-scoring.test.ts +0 -106
  215. package/src/__tests__/scope-inventory.test.ts +0 -200
  216. package/src/__tests__/scope-policy.test.ts +0 -69
  217. package/src/__tests__/session-checkpoint.test.ts +0 -259
  218. package/src/__tests__/session-distiller.test.ts +0 -611
  219. package/src/__tests__/session-gc.test.ts +0 -215
  220. package/src/__tests__/skill-engine.test.ts +0 -546
  221. package/src/__tests__/skill-promotion.test.ts +0 -349
  222. package/src/__tests__/store-scope-audit.test.ts +0 -81
  223. package/src/__tests__/supersede-chain.test.ts +0 -154
  224. package/src/__tests__/temporal-parser.test.ts +0 -241
  225. package/src/__tests__/temporal-validity.test.ts +0 -160
  226. package/src/__tests__/term-registry.test.ts +0 -121
  227. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  228. package/src/__tests__/topic-tag.test.ts +0 -113
  229. package/src/__tests__/trusted-memory.test.ts +0 -828
  230. package/src/__tests__/version-manager.test.ts +0 -213
  231. package/src/__tests__/workflow-observation.test.ts +0 -400
  232. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,588 +0,0 @@
1
- /**
2
- * 盲扫脚本 — 模块 4:开放式探索
3
- * 不带预设地分析 38K 条记忆的文本内容
4
- * 只读,不写入任何内容
5
- *
6
- * 用法:bun run scripts/blind-scan.ts [lancedb-dir]
7
- */
8
-
9
- import lancedb from "@lancedb/lancedb";
10
-
11
- const DB_PATH = process.argv[2] || "./data/lancedb";
12
- const TABLE_NAME = "memories";
13
-
14
- interface MemoryRow {
15
- id: string;
16
- text: string;
17
- vector: number[];
18
- category: string;
19
- scope: string;
20
- importance: number;
21
- timestamp: number;
22
- metadata?: string;
23
- }
24
-
25
- function cosineSimilarity(a: number[], b: number[]): number {
26
- let dot = 0, normA = 0, normB = 0;
27
- for (let i = 0; i < a.length; i++) {
28
- dot += a[i] * b[i];
29
- normA += a[i] * a[i];
30
- normB += b[i] * b[i];
31
- }
32
- return dot / (Math.sqrt(normA) * Math.sqrt(normB));
33
- }
34
-
35
- function formatDate(ts: number): string {
36
- return new Date(ts).toISOString().slice(0, 10);
37
- }
38
-
39
- // Simple CJK-aware tokenizer
40
- function tokenize(text: string): string[] {
41
- const tokens: string[] = [];
42
- // Remove URLs
43
- const cleaned = text.replace(/https?:\/\/\S+/g, " ");
44
- // Split into words (English) and characters (CJK)
45
- const parts = cleaned.match(/[\u4e00-\u9fff\u3400-\u4dbf]+|[a-zA-Z][a-zA-Z0-9_'-]+/g) || [];
46
- for (const part of parts) {
47
- if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(part)) {
48
- // CJK: bigrams
49
- for (let i = 0; i < part.length - 1; i++) {
50
- tokens.push(part.slice(i, i + 2));
51
- }
52
- } else if (part.length > 1) {
53
- tokens.push(part.toLowerCase());
54
- }
55
- }
56
- return tokens;
57
- }
58
-
59
- // Stopwords (minimal)
60
- const STOPWORDS = new Set([
61
- "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
62
- "have", "has", "had", "do", "does", "did", "will", "would", "could",
63
- "should", "may", "might", "shall", "can", "need", "must",
64
- "to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
65
- "into", "through", "during", "before", "after", "above", "below",
66
- "between", "out", "off", "over", "under", "again", "further",
67
- "then", "once", "here", "there", "when", "where", "why", "how",
68
- "all", "each", "every", "both", "few", "more", "most", "other",
69
- "some", "such", "no", "nor", "not", "only", "own", "same", "so",
70
- "than", "too", "very", "just", "because", "but", "and", "or", "if",
71
- "while", "about", "up", "its", "it", "this", "that", "these", "those",
72
- "he", "she", "they", "we", "you", "me", "him", "her", "us", "them",
73
- "my", "your", "his", "our", "their", "what", "which", "who", "whom",
74
- "also", "been", "like", "get", "got", "make", "made", "use", "used",
75
- ]);
76
-
77
- async function main() {
78
- console.log("\n🔍 盲扫报告 — RecallNest 开放式探索");
79
- console.log("=".repeat(70));
80
- console.log(`📅 时间: ${new Date().toISOString().slice(0, 19)}`);
81
- console.log(`📂 路径: ${DB_PATH}\n`);
82
-
83
- const db = await lancedb.connect(DB_PATH);
84
- const table = await db.openTable(TABLE_NAME);
85
-
86
- console.log("⏳ 读取全量数据...");
87
- const allRows: MemoryRow[] = await table
88
- .query()
89
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
90
- .toArray() as any;
91
-
92
- const total = allRows.length;
93
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
94
-
95
- const parsedRows = allRows.map(row => ({
96
- ...row,
97
- meta: row.metadata ? JSON.parse(row.metadata) : {},
98
- }));
99
-
100
- // ============================================
101
- // 1. 文本长度分布
102
- // ============================================
103
- console.log("📊 1. 文本长度分布");
104
- console.log("-".repeat(70));
105
- const lengths = parsedRows.map(r => r.text.length);
106
- lengths.sort((a, b) => a - b);
107
- const avgLen = lengths.reduce((s, l) => s + l, 0) / total;
108
- const medianLen = lengths[Math.floor(total / 2)];
109
- const p95Len = lengths[Math.floor(total * 0.95)];
110
- const p99Len = lengths[Math.floor(total * 0.99)];
111
-
112
- console.log(` 最短: ${lengths[0]} 字符`);
113
- console.log(` 中位: ${medianLen} 字符`);
114
- console.log(` 平均: ${Math.round(avgLen)} 字符`);
115
- console.log(` P95: ${p95Len} 字符`);
116
- console.log(` P99: ${p99Len} 字符`);
117
- console.log(` 最长: ${lengths[total - 1]} 字符`);
118
-
119
- // Length buckets
120
- const lenBuckets = new Map<string, number>();
121
- const lenRanges: Array<[number, number, string]> = [
122
- [0, 50, "0-50"],
123
- [50, 100, "50-100"],
124
- [100, 200, "100-200"],
125
- [200, 500, "200-500"],
126
- [500, 1000, "500-1K"],
127
- [1000, 2000, "1K-2K"],
128
- [2000, 5000, "2K-5K"],
129
- [5000, Infinity, "5K+"],
130
- ];
131
- for (const len of lengths) {
132
- for (const [lo, hi, label] of lenRanges) {
133
- if (len >= lo && len < hi) {
134
- lenBuckets.set(label, (lenBuckets.get(label) || 0) + 1);
135
- break;
136
- }
137
- }
138
- }
139
- console.log("\n 分布:");
140
- for (const [, , label] of lenRanges) {
141
- const count = lenBuckets.get(label) || 0;
142
- const pct = (count / total * 100).toFixed(1);
143
- const bar = "█".repeat(Math.round(parseFloat(pct) / 100 * 40));
144
- console.log(` ${label.padEnd(10)} ${count.toString().padStart(6)} (${pct.padStart(5)}%) ${bar}`);
145
- }
146
-
147
- // Extremely short texts (likely garbage)
148
- const ultraShort = parsedRows.filter(r => r.text.length < 20);
149
- if (ultraShort.length > 0) {
150
- console.log(`\n ⚠️ 超短文本 (<20 字符): ${ultraShort.length} 条`);
151
- console.log(" 示例:");
152
- for (const r of ultraShort.slice(0, 5)) {
153
- console.log(` [${r.category}] "${r.text}"`);
154
- }
155
- }
156
-
157
- // ============================================
158
- // 2. 语言混合分析
159
- // ============================================
160
- console.log("\n📊 2. 语言混合分析");
161
- console.log("-".repeat(70));
162
- let pureEn = 0, pureCn = 0, mixed = 0, other = 0;
163
- for (const row of parsedRows) {
164
- const hasCJK = /[\u4e00-\u9fff]/.test(row.text);
165
- const hasEn = /[a-zA-Z]{3,}/.test(row.text);
166
- if (hasCJK && hasEn) mixed++;
167
- else if (hasCJK) pureCn++;
168
- else if (hasEn) pureEn++;
169
- else other++;
170
- }
171
- console.log(` 纯英文: ${pureEn.toLocaleString()} (${(pureEn/total*100).toFixed(1)}%)`);
172
- console.log(` 纯中文: ${pureCn.toLocaleString()} (${(pureCn/total*100).toFixed(1)}%)`);
173
- console.log(` 中英混合: ${mixed.toLocaleString()} (${(mixed/total*100).toFixed(1)}%)`);
174
- console.log(` 其他: ${other.toLocaleString()} (${(other/total*100).toFixed(1)}%)`);
175
-
176
- // ============================================
177
- // 3. 高频词分析(分语言)
178
- // ============================================
179
- console.log("\n📊 3. 高频词分析");
180
- console.log("-".repeat(70));
181
- const wordFreq = new Map<string, number>();
182
- for (const row of parsedRows) {
183
- const tokens = tokenize(row.text);
184
- for (const t of tokens) {
185
- if (!STOPWORDS.has(t) && t.length > 1) {
186
- wordFreq.set(t, (wordFreq.get(t) || 0) + 1);
187
- }
188
- }
189
- }
190
- const topWords = [...wordFreq.entries()]
191
- .sort((a, b) => b[1] - a[1])
192
- .slice(0, 40);
193
-
194
- console.log(" Top 40 词频(去停用词):");
195
- for (let i = 0; i < topWords.length; i++) {
196
- const [word, count] = topWords[i];
197
- const pctOfDocs = (count / total * 100).toFixed(1);
198
- console.log(` ${(i+1).toString().padStart(2)}. ${word.padEnd(20)} ${count.toString().padStart(6)} (${pctOfDocs}% 文档)`);
199
- }
200
-
201
- // ============================================
202
- // 4. 文本前缀聚类(发现模板化内容)
203
- // ============================================
204
- console.log("\n📊 4. 文本前缀聚类(发现模板化内容)");
205
- console.log("-".repeat(70));
206
- const prefixMap = new Map<string, number>();
207
- for (const row of parsedRows) {
208
- // Take first 40 chars as prefix fingerprint
209
- const prefix = row.text.slice(0, 40).trim();
210
- prefixMap.set(prefix, (prefixMap.get(prefix) || 0) + 1);
211
- }
212
- const repeatedPrefixes = [...prefixMap.entries()]
213
- .filter(([, count]) => count >= 3)
214
- .sort((a, b) => b[1] - a[1]);
215
-
216
- console.log(` 独立前缀数: ${prefixMap.size.toLocaleString()}`);
217
- console.log(` 重复 ≥3 次的前缀: ${repeatedPrefixes.length}`);
218
-
219
- if (repeatedPrefixes.length > 0) {
220
- console.log("\n Top 20 重复前缀:");
221
- for (const [prefix, count] of repeatedPrefixes.slice(0, 20)) {
222
- console.log(` [×${count.toString().padStart(4)}] ${prefix}...`);
223
- }
224
- }
225
-
226
- // ============================================
227
- // 5. Scope × Category 交叉分析
228
- // ============================================
229
- console.log("\n📊 5. Scope × Category 交叉分析(Top 10 Scope)");
230
- console.log("-".repeat(70));
231
- const scopeCat = new Map<string, Map<string, number>>();
232
- for (const row of parsedRows) {
233
- const s = row.scope || "(empty)";
234
- if (!scopeCat.has(s)) scopeCat.set(s, new Map());
235
- const catMap = scopeCat.get(s)!;
236
- catMap.set(row.category, (catMap.get(row.category) || 0) + 1);
237
- }
238
- const topScopes = [...scopeCat.entries()]
239
- .map(([scope, cats]) => {
240
- const scopeTotal = [...cats.values()].reduce((s, c) => s + c, 0);
241
- return { scope, total: scopeTotal, cats };
242
- })
243
- .sort((a, b) => b.total - a.total)
244
- .slice(0, 10);
245
-
246
- const allCats = [...new Set(parsedRows.map(r => r.category))].sort();
247
- // Header
248
- let header = " Scope".padEnd(35);
249
- for (const cat of allCats) header += cat.slice(0, 6).padStart(8);
250
- header += " Total".padStart(8);
251
- console.log(header);
252
-
253
- for (const { scope, total: scopeTotal, cats } of topScopes) {
254
- const label = scope.length > 30 ? scope.slice(0, 27) + "..." : scope;
255
- let line = ` ${label.padEnd(33)}`;
256
- for (const cat of allCats) {
257
- line += (cats.get(cat) || 0).toString().padStart(8);
258
- }
259
- line += scopeTotal.toString().padStart(8);
260
- console.log(line);
261
- }
262
-
263
- // ============================================
264
- // 6. 时间模式分析
265
- // ============================================
266
- console.log("\n📊 6. 时间模式分析");
267
- console.log("-".repeat(70));
268
-
269
- // By month
270
- const monthMap = new Map<string, number>();
271
- // By hour of day
272
- const hourMap = new Map<number, number>();
273
- // By day of week
274
- const dowMap = new Map<number, number>();
275
- const DOW_NAMES = ["周日", "周一", "周二", "周三", "周四", "周五", "周六"];
276
-
277
- for (const row of parsedRows) {
278
- const d = new Date(row.timestamp);
279
- const month = d.toISOString().slice(0, 7);
280
- monthMap.set(month, (monthMap.get(month) || 0) + 1);
281
- hourMap.set(d.getHours(), (hourMap.get(d.getHours()) || 0) + 1);
282
- dowMap.set(d.getDay(), (dowMap.get(d.getDay()) || 0) + 1);
283
- }
284
-
285
- // Monthly trend
286
- const months = [...monthMap.entries()].sort();
287
- console.log(" 月度趋势:");
288
- const maxMonthCount = Math.max(...months.map(m => m[1]));
289
- for (const [month, count] of months) {
290
- const bar = "█".repeat(Math.round(count / maxMonthCount * 30));
291
- console.log(` ${month} ${count.toString().padStart(6)} ${bar}`);
292
- }
293
-
294
- // Hourly pattern
295
- console.log("\n 小时分布:");
296
- const maxHourCount = Math.max(...[...hourMap.values()]);
297
- for (let h = 0; h < 24; h++) {
298
- const count = hourMap.get(h) || 0;
299
- const bar = "█".repeat(Math.round(count / maxHourCount * 20));
300
- console.log(` ${h.toString().padStart(2)}:00 ${count.toString().padStart(5)} ${bar}`);
301
- }
302
-
303
- // Day of week
304
- console.log("\n 星期分布:");
305
- for (let d = 0; d < 7; d++) {
306
- const count = dowMap.get(d) || 0;
307
- const pct = (count / total * 100).toFixed(1);
308
- console.log(` ${DOW_NAMES[d]} ${count.toString().padStart(6)} (${pct}%)`);
309
- }
310
-
311
- // ============================================
312
- // 7. 过时内容检测
313
- // ============================================
314
- console.log("\n📊 7. 过时内容检测");
315
- console.log("-".repeat(70));
316
-
317
- // Look for date references in text
318
- const yearRefs = new Map<string, number>();
319
- for (const row of parsedRows) {
320
- const yearPattern = /20(2[0-6])/g;
321
- let match;
322
- while ((match = yearPattern.exec(row.text)) !== null) {
323
- const year = `20${match[1]}`;
324
- yearRefs.set(year, (yearRefs.get(year) || 0) + 1);
325
- }
326
- }
327
- console.log(" 文本中提到的年份:");
328
- for (const [year, count] of [...yearRefs.entries()].sort()) {
329
- console.log(` ${year}: ${count.toLocaleString()} 次`);
330
- }
331
-
332
- // Look for potentially outdated keywords
333
- const outdatedKeywords = [
334
- "deprecated", "removed", "old", "legacy", "TODO", "FIXME", "HACK",
335
- "temporary", "workaround", "临时", "废弃", "已删除", "待修复",
336
- ];
337
- console.log("\n 过时关键词扫描:");
338
- for (const kw of outdatedKeywords) {
339
- const count = parsedRows.filter(r =>
340
- r.text.toLowerCase().includes(kw.toLowerCase())
341
- ).length;
342
- if (count > 0) {
343
- console.log(` "${kw}": ${count} 条`);
344
- }
345
- }
346
-
347
- // ============================================
348
- // 8. 文本精确重复检测
349
- // ============================================
350
- console.log("\n📊 8. 精确/近似文本重复");
351
- console.log("-".repeat(70));
352
-
353
- // Exact text duplicates
354
- const textMap = new Map<string, number>();
355
- for (const row of parsedRows) {
356
- const normalized = row.text.trim().toLowerCase();
357
- textMap.set(normalized, (textMap.get(normalized) || 0) + 1);
358
- }
359
- const exactDups = [...textMap.entries()].filter(([, count]) => count > 1);
360
- const totalDupRows = exactDups.reduce((s, [, c]) => s + c, 0);
361
- console.log(` 精确重复组: ${exactDups.length}`);
362
- console.log(` 重复行总数: ${totalDupRows} / ${total} (${(totalDupRows/total*100).toFixed(1)}%)`);
363
-
364
- if (exactDups.length > 0) {
365
- const topDups = exactDups.sort((a, b) => b[1] - a[1]).slice(0, 10);
366
- console.log("\n Top 10 精确重复:");
367
- for (const [text, count] of topDups) {
368
- console.log(` [×${count}] ${text.slice(0, 80)}...`);
369
- }
370
- }
371
-
372
- // Text fingerprint duplicates (first 100 chars match)
373
- const fpMap = new Map<string, number>();
374
- for (const row of parsedRows) {
375
- const fp = row.text.trim().slice(0, 100).toLowerCase();
376
- fpMap.set(fp, (fpMap.get(fp) || 0) + 1);
377
- }
378
- const fpDups = [...fpMap.entries()].filter(([, count]) => count > 1);
379
- const fpDupRows = fpDups.reduce((s, [, c]) => s + c, 0);
380
- console.log(`\n 前100字符重复组: ${fpDups.length}`);
381
- console.log(` 相关行总数: ${fpDupRows} / ${total} (${(fpDupRows/total*100).toFixed(1)}%)`);
382
-
383
- // ============================================
384
- // 9. Scope 生态分析
385
- // ============================================
386
- console.log("\n📊 9. Scope 生态分析");
387
- console.log("-".repeat(70));
388
-
389
- const scopeCounts = new Map<string, number>();
390
- for (const row of parsedRows) {
391
- const s = row.scope || "(empty)";
392
- scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
393
- }
394
- const totalScopes = scopeCounts.size;
395
- const singletonScopes = [...scopeCounts.entries()].filter(([, c]) => c === 1).length;
396
- const tinyScopes = [...scopeCounts.entries()].filter(([, c]) => c <= 5 && c > 1).length;
397
-
398
- console.log(` 总 scope 数: ${totalScopes}`);
399
- console.log(` 单条 scope(孤儿): ${singletonScopes} (${(singletonScopes/totalScopes*100).toFixed(1)}%)`);
400
- console.log(` 2-5 条 scope: ${tinyScopes}`);
401
- console.log(` Top 1 scope 占比: ${(([...scopeCounts.values()].sort((a,b)=>b-a)[0] / total) * 100).toFixed(1)}%`);
402
-
403
- // ============================================
404
- // 10. 内容主题聚类(基于 embedding 采样)
405
- // ============================================
406
- const clusterSampleSize = 2000;
407
- console.log(`\n📊 10. Embedding 聚类采样(${clusterSampleSize} 条,K-Means k=8)`);
408
- console.log("-".repeat(70));
409
- console.log(" ⏳ 读取向量数据...");
410
-
411
- // Read vectors for sample
412
- const sampleWithVectors: MemoryRow[] = await table
413
- .query()
414
- .select(["id", "text", "vector", "category", "scope", "timestamp"])
415
- .limit(clusterSampleSize)
416
- .toArray() as any;
417
-
418
- // Simple K-Means
419
- const K = 8;
420
- const dim = sampleWithVectors[0]?.vector?.length || 0;
421
- if (dim === 0) {
422
- console.log(" ⚠️ 无向量数据,跳过聚类");
423
- } else {
424
- console.log(` 向量维度: ${dim}`);
425
-
426
- // Initialize centroids randomly
427
- const centroids: number[][] = [];
428
- const usedSet = new Set<number>();
429
- for (let i = 0; i < K; i++) {
430
- let idx: number;
431
- do { idx = Math.floor(Math.random() * sampleWithVectors.length); } while (usedSet.has(idx));
432
- usedSet.add(idx);
433
- centroids.push([...sampleWithVectors[idx].vector]);
434
- }
435
-
436
- // Run K-Means for 20 iterations
437
- const assignments = new Array(sampleWithVectors.length).fill(0);
438
- for (let iter = 0; iter < 20; iter++) {
439
- // Assign
440
- for (let i = 0; i < sampleWithVectors.length; i++) {
441
- let bestK = 0, bestSim = -1;
442
- for (let k = 0; k < K; k++) {
443
- const sim = cosineSimilarity(sampleWithVectors[i].vector, centroids[k]);
444
- if (sim > bestSim) { bestSim = sim; bestK = k; }
445
- }
446
- assignments[i] = bestK;
447
- }
448
- // Update centroids
449
- for (let k = 0; k < K; k++) {
450
- const members = sampleWithVectors.filter((_, i) => assignments[i] === k);
451
- if (members.length === 0) continue;
452
- for (let d = 0; d < dim; d++) {
453
- centroids[k][d] = members.reduce((s, m) => s + m.vector[d], 0) / members.length;
454
- }
455
- }
456
- }
457
-
458
- // Analyze clusters
459
- const clusters: Map<number, typeof sampleWithVectors> = new Map();
460
- for (let i = 0; i < sampleWithVectors.length; i++) {
461
- if (!clusters.has(assignments[i])) clusters.set(assignments[i], []);
462
- clusters.get(assignments[i])!.push(sampleWithVectors[i]);
463
- }
464
-
465
- console.log(`\n 聚类结果 (K=${K}):`);
466
- const sortedClusters = [...clusters.entries()].sort((a, b) => b[1].length - a[1].length);
467
-
468
- for (const [k, members] of sortedClusters) {
469
- console.log(`\n ── Cluster ${k} (${members.length} 条, ${(members.length/sampleWithVectors.length*100).toFixed(1)}%) ──`);
470
-
471
- // Category distribution within cluster
472
- const clCatMap = new Map<string, number>();
473
- for (const m of members) {
474
- clCatMap.set(m.category, (clCatMap.get(m.category) || 0) + 1);
475
- }
476
- const catStr = [...clCatMap.entries()]
477
- .sort((a, b) => b[1] - a[1])
478
- .map(([c, n]) => `${c}:${n}`)
479
- .join(", ");
480
- console.log(` 类别: ${catStr}`);
481
-
482
- // Top scopes
483
- const clScopeMap = new Map<string, number>();
484
- for (const m of members) {
485
- const s = m.scope || "(empty)";
486
- clScopeMap.set(s, (clScopeMap.get(s) || 0) + 1);
487
- }
488
- const topClScopes = [...clScopeMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
489
- console.log(` Top scope: ${topClScopes.map(([s, n]) => `${s.slice(0,30)}(${n})`).join(", ")}`);
490
-
491
- // Top keywords
492
- const clWordFreq = new Map<string, number>();
493
- for (const m of members) {
494
- const tokens = tokenize(m.text);
495
- const seen = new Set<string>();
496
- for (const t of tokens) {
497
- if (!STOPWORDS.has(t) && t.length > 1 && !seen.has(t)) {
498
- clWordFreq.set(t, (clWordFreq.get(t) || 0) + 1);
499
- seen.add(t);
500
- }
501
- }
502
- }
503
- const topClWords = [...clWordFreq.entries()]
504
- .sort((a, b) => b[1] - a[1])
505
- .slice(0, 8)
506
- .map(([w]) => w)
507
- .join(", ");
508
- console.log(` 关键词: ${topClWords}`);
509
-
510
- // Sample texts
511
- console.log(` 示例:`);
512
- const samples = members.slice(0, 3);
513
- for (const s of samples) {
514
- console.log(` - ${s.text.slice(0, 100)}...`);
515
- }
516
- }
517
- }
518
-
519
- // ============================================
520
- // 11. 异常值检测
521
- // ============================================
522
- console.log("\n📊 11. 异常值检测");
523
- console.log("-".repeat(70));
524
-
525
- // Empty or near-empty scope
526
- const emptyScope = parsedRows.filter(r => !r.scope || r.scope.trim() === "").length;
527
- console.log(` 空 scope: ${emptyScope} 条 (${(emptyScope/total*100).toFixed(1)}%)`);
528
-
529
- // Suspiciously high importance + zero access
530
- const highImpZeroAccess = parsedRows.filter(r => {
531
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
532
- return r.importance >= 0.8 && ac === 0;
533
- }).length;
534
- console.log(` 高 importance(≥0.8) + 0 访问: ${highImpZeroAccess} 条`);
535
-
536
- // Very old memories that are still "working" tier
537
- const oldWorking = parsedRows.filter(r => {
538
- const tier = r.meta.tier ?? "unknown";
539
- const agedays = (Date.now() - r.timestamp) / (1000*60*60*24);
540
- return tier === "working" && agedays > 60;
541
- }).length;
542
- console.log(` >60天的 working tier 记忆: ${oldWorking} 条(应降级?)`);
543
-
544
- // Memories with no metadata
545
- const noMeta = parsedRows.filter(r => !r.metadata || r.metadata === "{}").length;
546
- console.log(` 无 metadata: ${noMeta} 条`);
547
-
548
- // ============================================
549
- // 综合发现
550
- // ============================================
551
- console.log("\n" + "=".repeat(70));
552
- console.log("🔬 盲扫综合发现");
553
- console.log("=".repeat(70));
554
-
555
- const findings: string[] = [];
556
-
557
- if (exactDups.length > 0) {
558
- findings.push(`📌 精确重复 ${exactDups.length} 组 / ${totalDupRows} 条 — 有清理空间`);
559
- }
560
- if (ultraShort.length > 0) {
561
- findings.push(`📌 超短文本 (<20字符) ${ultraShort.length} 条 — 可能是垃圾数据`);
562
- }
563
- if (singletonScopes > totalScopes * 0.5) {
564
- findings.push(`📌 ${singletonScopes}/${totalScopes} scope 只有 1 条记忆 — scope 碎片化严重`);
565
- }
566
- if (highImpZeroAccess > 100) {
567
- findings.push(`📌 ${highImpZeroAccess} 条高重要性记忆从未被访问 — importance 标记可能虚高`);
568
- }
569
- if (mixed / total > 0.3) {
570
- findings.push(`📌 ${(mixed/total*100).toFixed(1)}% 记忆中英混合 — 可能影响检索一致性`);
571
- }
572
- if (emptyScope > total * 0.1) {
573
- findings.push(`📌 ${(emptyScope/total*100).toFixed(1)}% 记忆无 scope — 检索时无法按项目过滤`);
574
- }
575
-
576
- for (const f of findings) {
577
- console.log(` ${f}`);
578
- }
579
-
580
- if (findings.length === 0) {
581
- console.log(" ✅ 未发现明显异常模式");
582
- }
583
-
584
- console.log("\n" + "=".repeat(70));
585
- console.log("📋 盲扫完成\n");
586
- }
587
-
588
- main().catch(console.error);